diff --git a/Cargo.lock b/Cargo.lock index 5dbe4ce6..4c4911cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1174,6 +1174,28 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-runtime" version = "1.7.1" @@ -1699,6 +1721,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -1722,13 +1746,15 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -2025,6 +2051,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version 0.4.1", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2405,6 +2440,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "dipper-producer" +version = "0.1.0" +dependencies = [ + "async-trait", + "chrono", + "fs-err", + "prost 0.14.4", + "prost-build", + "rskafka", + "rustls", + "rustls-pemfile 2.2.0", + "serde", + "thegraph-core", + "thiserror 2.0.18", + "tokio", + "tracing", + "uuid", + "webpki-roots", +] + [[package]] name = "dipper-rpc" version = "0.1.0" @@ -2430,6 +2486,7 @@ dependencies = [ "dipper-iisa", "dipper-pgmq", "dipper-pgregistry", + "dipper-producer", "dipper-rpc", "fake", "futures-lite", @@ -2798,6 +2855,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +dependencies = [ + "autocfg", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -3630,7 +3702,7 @@ dependencies = [ "futures", "graph-networks-registry 0.6.3", "ipfs-api-backend-hyper", - "prost 0.14.3", + "prost 0.14.4", "serde", "serde_yaml", "thegraph-core", @@ -3682,6 +3754,12 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +[[package]] +name = "integer-encoding" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c00403deb17c3221a1fe4fb571b9ed0370b3dcd116553c77fa294a3d918699" + [[package]] name = "ipfs-api-backend-hyper" version = "0.6.0" @@ -3811,6 +3889,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.85" @@ -4144,6 +4232,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "macro-string" version = "0.1.4" @@ -4965,19 +5072,19 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", - "prost-derive 0.14.3", + "prost-derive 0.14.4", ] [[package]] name = "prost-build" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", "itertools 0.14.0", @@ -4985,8 +5092,8 @@ dependencies = [ "multimap", "petgraph", "prettyplease", - "prost 0.14.3", - "prost-types 0.14.3", + "prost 0.14.4", + "prost-types 0.14.4", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", @@ -5022,9 +5129,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools 0.14.0", @@ -5053,11 +5160,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "prost 0.14.3", + "prost 0.14.4", ] [[package]] @@ -5350,7 +5457,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -5484,6 +5591,49 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rsasl" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed828a88913fd477c73bc3768b05d4b335ee775e29f2397bb59b9a4dd69ffb83" +dependencies = [ + "base64 0.22.1", + "digest 0.10.7", + "hmac", + "pbkdf2", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "stringprep", + "thiserror 2.0.18", +] + +[[package]] +name = "rskafka" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "849b87417a191e37e16b8893eba8928bbd10e1989e6b57c4f8531549eaa29bdc" +dependencies = [ + "bytes", + "chrono", + "crc32c", + "flate2", + "futures", + "integer-encoding", + "lz4", + "parking_lot", + "rand 0.8.5", + "rsasl", + "rustls", + "snap", + "thiserror 1.0.69", + "tokio", + "tokio-rustls", + "tracing", + "zstd", +] + [[package]] name = "ruint" version = "1.17.2" @@ -5573,10 +5723,11 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -5607,6 +5758,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -5650,6 +5810,7 @@ version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5769,8 +5930,8 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "662c7f8e99d46c9d3a87561d771a970c29efaccbab4bbdc6ab65d099d2358077" dependencies = [ - "prost 0.14.3", - "prost-types 0.14.3", + "prost 0.14.4", + "prost-types 0.14.4", "serde", "serde_json", "zeroize", @@ -6068,6 +6229,12 @@ dependencies = [ "serde", ] +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "snmalloc-rs" version = "0.3.8" @@ -6862,7 +7029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" dependencies = [ "bytes", - "prost 0.14.3", + "prost 0.14.4", "tonic 0.14.5", ] @@ -6875,7 +7042,7 @@ dependencies = [ "prettyplease", "proc-macro2", "prost-build", - "prost-types 0.14.3", + "prost-types 0.14.4", "quote", "syn 2.0.116", "tempfile", @@ -7440,9 +7607,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] @@ -8134,3 +8301,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 977d8e1d..19ea8a2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "dipper-iisa", "dipper-pgmq", "dipper-pgregistry", + "dipper-producer", "dipper-rpc", ] diff --git a/bin/dipper-service/Cargo.toml b/bin/dipper-service/Cargo.toml index 4d235892..67375c3e 100644 --- a/bin/dipper-service/Cargo.toml +++ b/bin/dipper-service/Cargo.toml @@ -12,6 +12,7 @@ dipper-core = { version = "0.1.0", path = "../../dipper-core" } dipper-iisa = { path = "../../dipper-iisa" } dipper-pgmq = { path = "../../dipper-pgmq" } dipper-pgregistry = { path = "../../dipper-pgregistry" } +dipper-producer = { path = "../../dipper-producer" } dipper-rpc = { version = "0.1.0", path = "../../dipper-rpc", features = ["admin-rpc", "indexer-rpc"] } futures-lite = { version = "2.4.0", default-features = false } graph-networks-registry.workspace = true diff --git a/bin/dipper-service/src/admin_rpc_server/context.rs b/bin/dipper-service/src/admin_rpc_server/context.rs index c4b5d478..9e2b44fb 100644 --- a/bin/dipper-service/src/admin_rpc_server/context.rs +++ b/bin/dipper-service/src/admin_rpc_server/context.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeSet, sync::Arc}; use dipper_core::state::FromState; +use dipper_producer::events::SubgraphIndexingAgreementEventsProducer; use thegraph_core::alloy::primitives::Address; use super::handlers::{IndexingAgreementsCtx, IndexingRequestsCtx}; @@ -23,6 +24,10 @@ pub struct Ctx { /// The message queue worker pub worker: W, + + /// Subgraph Indexing Agreements Events emitter for sending on the topic, if configured and enabled + pub subgraph_indexing_agreements_events_emitter: + Arc, } impl FromState> for IndexingRequestsCtx @@ -37,6 +42,9 @@ where registry: ctx.registry.clone(), worker: ctx.worker.clone(), max_candidates: ctx.max_candidates, + subgraph_indexing_agreements_events_emitter: ctx + .subgraph_indexing_agreements_events_emitter + .clone(), } } } diff --git a/bin/dipper-service/src/admin_rpc_server/handlers/indexing_requests.rs b/bin/dipper-service/src/admin_rpc_server/handlers/indexing_requests.rs index 1b1479da..e5c26605 100644 --- a/bin/dipper-service/src/admin_rpc_server/handlers/indexing_requests.rs +++ b/bin/dipper-service/src/admin_rpc_server/handlers/indexing_requests.rs @@ -2,6 +2,7 @@ use std::{collections::BTreeSet, sync::Arc}; use async_trait::async_trait; use dipper_core::{ids::IndexingRequestId, state::FromState}; +use dipper_producer::{events::SubgraphIndexingAgreementEventsProducer, proto}; use dipper_rpc::admin::{ SignedMessage, indexing_requests::{ @@ -31,6 +32,8 @@ pub struct Ctx { pub registry: R, pub worker: W, pub max_candidates: usize, + pub subgraph_indexing_agreements_events_emitter: + Arc, } pub struct RpcServerImpl(Ctx); @@ -130,6 +133,21 @@ where num_candidates, "Inserted new indexing request" ); + + // A new request was received: emit the lifecycle event. Only the + // `Inserted` outcome is a genuinely new request; `Updated`/`Canceled` + // are later transitions in the lifecycle, not "request received". + // `the_graph_network` is the protocol network (signer chain id), not + // the deployment's data-source `chain_id`. + self.subgraph_indexing_agreements_events_emitter + .produce_subgraph_indexing_agreement_request_received( + deployment_id, + self.signer.chain_id(), + proto::SubgraphIndexingAgreementRequestReceived { + agreements_requested: num_candidates as i32, + }, + ); + (Some(id), Some(num_candidates)) } SetTargetOutcome::Updated { @@ -230,3 +248,270 @@ fn into_indexing_request_status(status: IndexingRequestRecordStatus) -> Indexing IndexingRequestRecordStatus::Canceled => IndexingRequestStatus::Canceled, } } + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use dipper_core::ids::{IndexingAgreementId, IndexingRequestId}; + use dipper_rpc::admin::indexing_requests::SetIndexingTargetCandidates; + use thegraph_core::{ + DeploymentId, + alloy::{ + primitives::{Address, ChainId}, + signers::local::PrivateKeySigner, + }, + deployment_id, + signed_message::sign, + }; + use url::Url; + + use super::*; + use crate::{ + registry::Result as RegistryResult, + test_support::{CapturedEvent, CapturingEventsProducer}, + worker::queue::JobId, + }; + + /// The protocol (signer) chain id. Deliberately different from the + /// request's deployment `chain_id` so the assertion that the emitted + /// event carries the *signer* chain id is meaningful. + const SIGNER_CHAIN_ID: ChainId = 42161; + + /// The request's deployment chain id (a data-source chain), intentionally + /// distinct from [`SIGNER_CHAIN_ID`]. + const DEPLOYMENT_CHAIN_ID: ChainId = 1; + + /// A registry whose `set_indexing_target_candidates` returns a configured + /// outcome. All other trait methods are unused by the handler path under + /// test and therefore `unimplemented!()`. + #[derive(Clone)] + struct MockRegistry { + outcome: SetTargetOutcome, + } + + #[async_trait] + impl IndexingRequestRegistry for MockRegistry { + async fn set_indexing_target_candidates( + &self, + _requested_by: Address, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _num_candidates: usize, + ) -> RegistryResult { + Ok(self.outcome.clone()) + } + + async fn get_all_indexing_requests(&self) -> RegistryResult> { + unimplemented!() + } + + async fn get_indexing_request_by_id( + &self, + _id: &IndexingRequestId, + ) -> RegistryResult> { + unimplemented!() + } + + async fn get_indexing_requests_by_deployment_id( + &self, + _deployment_id: &DeploymentId, + ) -> RegistryResult> { + unimplemented!() + } + + async fn get_open_indexing_requests_for_reassessment( + &self, + _min_age_seconds: i64, + _batch_size: i64, + ) -> RegistryResult> { + unimplemented!() + } + } + + /// A worker that accepts `reassess_indexing_request` and returns a default + /// `JobId`. All other queue methods are unused by this path. + #[derive(Clone)] + struct MockWorker; + + #[async_trait] + impl WorkerQueue for MockWorker { + async fn send_indexing_agreement_proposal( + &self, + _candidate_url: Url, + _agreement_id: IndexingAgreementId, + _indexing_request_id: IndexingRequestId, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + unimplemented!() + } + + async fn reassess_indexing_request( + &self, + _indexing_request_id: IndexingRequestId, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _num_candidates: usize, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + Ok(JobId::default()) + } + + async fn cancel_rejected_agreement_on_chain( + &self, + _agreement_id: IndexingAgreementId, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + unimplemented!() + } + + async fn submit_offer( + &self, + _agreement_id: IndexingAgreementId, + _indexing_request_id: IndexingRequestId, + _indexer_url: Url, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + unimplemented!() + } + } + + /// Build a signed `set_indexing_target_candidates` request using the + /// canonical `thegraph_core::signed_message::sign` helper and the admin + /// EIP-712 domain, returning both the signer (so its address can be + /// allowlisted) and the wrapped `SignedMessage`. + fn signed_request( + deployment_id: DeploymentId, + chain_id: ChainId, + num_candidates: Option, + ) -> (PrivateKeySigner, SignedMessage) { + let signer = PrivateKeySigner::random(); + let domain = dipper_rpc::admin::eip712_domain(); + let message = SetIndexingTargetCandidates { + deployment_id, + chain_id, + num_candidates, + }; + let inner = sign(&signer, &domain, message).expect("signing failed"); + (signer, inner.into()) + } + + /// Assemble an `RpcServerImpl` whose signer's chain id is + /// [`SIGNER_CHAIN_ID`], whose allowlist contains `allowed`, and whose + /// registry returns `outcome`. Returns the server and the shared events + /// capture for assertions. + fn server( + allowed: Address, + outcome: SetTargetOutcome, + ) -> ( + RpcServerImpl, + CapturingEventsProducer, + ) { + let domain = dipper_rpc::admin::eip712_domain(); + // The signer's own address is irrelevant to recovery here (the handler + // recovers the *message* signer), so any address works. + let signer = Eip712Signer::new(SIGNER_CHAIN_ID, domain); + + let events = CapturingEventsProducer::new(); + + let ctx = Ctx { + signer: Arc::new(signer), + gateway_operator_allowlist: Arc::new(BTreeSet::from([allowed])), + registry: MockRegistry { outcome }, + worker: MockWorker, + max_candidates: 10, + subgraph_indexing_agreements_events_emitter: Arc::new(events.clone()), + }; + + (RpcServerImpl(ctx), events) + } + + #[tokio::test] + async fn inserted_emits_single_request_received_with_signer_chain_id() { + let deployment = deployment_id!("QmUzRg2HHMpbgf6Q4VHKNDbtBEJnyp5JWCh2gUX9AV6jXv"); + let num_candidates = 7usize; + + let (signer, req) = signed_request(deployment, DEPLOYMENT_CHAIN_ID, Some(num_candidates)); + let (server, events) = server( + signer.address(), + SetTargetOutcome::Inserted { + id: IndexingRequestId::new(), + }, + ); + + let result = server.set_indexing_target_candidates(req).await; + assert!(result.is_ok(), "handler returned an error: {result:?}"); + + let captured = events.events(); + assert_eq!(captured.len(), 1, "expected exactly one emitted event"); + + match &captured[0] { + CapturedEvent::RequestReceived { + deployment: ev_deployment, + chain_id: ev_chain_id, + event, + } => { + assert_eq!(*ev_deployment, deployment, "deployment id mismatch"); + assert_eq!( + *ev_chain_id, SIGNER_CHAIN_ID, + "event must carry the signer (protocol) chain id, not the deployment chain id" + ); + assert_ne!( + *ev_chain_id, DEPLOYMENT_CHAIN_ID, + "event chain id must not be the request's deployment chain id" + ); + assert_eq!( + event.agreements_requested, num_candidates as i32, + "agreements_requested mismatch" + ); + } + other => panic!("expected RequestReceived, got {other:?}"), + } + } + + #[tokio::test] + async fn noop_emits_no_event() { + assert_no_event_for(SetTargetOutcome::NoOp { + id: IndexingRequestId::new(), + }) + .await; + } + + #[tokio::test] + async fn updated_emits_no_event() { + assert_no_event_for(SetTargetOutcome::Updated { + id: IndexingRequestId::new(), + new_num_candidates: 3, + }) + .await; + } + + #[tokio::test] + async fn canceled_emits_no_event() { + assert_no_event_for(SetTargetOutcome::Canceled { + id: IndexingRequestId::new(), + }) + .await; + } + + /// Drive the handler with `outcome` and assert no lifecycle event is + /// emitted. `Updated`/`Canceled`/`NoOp` are not "request received". + async fn assert_no_event_for(outcome: SetTargetOutcome) { + let deployment = deployment_id!("QmUzRg2HHMpbgf6Q4VHKNDbtBEJnyp5JWCh2gUX9AV6jXv"); + + let (signer, req) = signed_request(deployment, DEPLOYMENT_CHAIN_ID, Some(5)); + let (server, events) = server(signer.address(), outcome); + + let result = server.set_indexing_target_candidates(req).await; + assert!(result.is_ok(), "handler returned an error: {result:?}"); + + assert!( + events.events().is_empty(), + "expected no emitted events, got {:?}", + events.events() + ); + } +} diff --git a/bin/dipper-service/src/cancel_dispatch.rs b/bin/dipper-service/src/cancel_dispatch.rs index 8797e92b..cd068be4 100644 --- a/bin/dipper-service/src/cancel_dispatch.rs +++ b/bin/dipper-service/src/cancel_dispatch.rs @@ -202,6 +202,7 @@ mod tests { subgraph_deployment_id: deployment_id, protocol_network: 1u64, chain_id: 1u64, + proposed_at: 0, }, }, last_block_height: None, diff --git a/bin/dipper-service/src/config.rs b/bin/dipper-service/src/config.rs index e8d80b5b..d3cc88b6 100644 --- a/bin/dipper-service/src/config.rs +++ b/bin/dipper-service/src/config.rs @@ -2,12 +2,14 @@ use std::{ collections::{BTreeMap, BTreeSet}, + num::NonZeroUsize, path::Path, sync::Arc, time::Duration, }; use dipper_core::config::{Hidden, HiddenSecretKeyAsHexStr}; +use dipper_producer::kafka::KafkaConfig; use serde_with::serde_as; use thegraph_core::alloy::{ primitives::{Address, ChainId, U256}, @@ -80,6 +82,9 @@ pub struct Config { /// The chain client configuration (for sending on-chain transactions) #[serde(default)] pub chain_client: Option, + /// Events configuration for sending dipper events on the configured topic for streaming + #[serde(default)] + pub event_streaming_config: Option, /// Number of concurrent worker loops draining the job queue (default: 8). /// Each loop can hold up to three pooled DB connections at once and shares /// the pool with the registry and background services; size accordingly. @@ -275,6 +280,20 @@ pub struct ExpirationConfig { /// Maximum agreements to process per cycle (default: 100) #[serde(default = "default_expiration_batch_size")] pub batch_size: i64, + + /// Grace period (chain seconds) held back past the deadline before an + /// agreement is marked `Expired` (default: 300s). + /// + /// The local `Created` row lags the chain, so an indexer's on-chain accept + /// within the deadline may not be reflected locally the instant the deadline + /// passes. Waiting this margin past the deadline lets the chain_listener sync + /// a within-deadline accept (flipping the row to `AcceptedOnChain`) before we + /// consider it expired -- preventing a premature `expired` event that would + /// contradict a subsequent `accepted`. Set to cover the worst-case subgraph + /// sync lag. + #[serde_as(as = "serde_with::DurationSeconds")] + #[serde(default = "default_expiration_grace")] + pub grace: Duration, } fn default_expiration_enabled() -> bool { @@ -289,6 +308,10 @@ fn default_expiration_batch_size() -> i64 { 100 } +fn default_expiration_grace() -> Duration { + Duration::from_secs(300) +} + fn default_worker_concurrency() -> usize { 8 } @@ -299,6 +322,7 @@ impl Default for ExpirationConfig { enabled: default_expiration_enabled(), interval: default_expiration_interval(), batch_size: default_expiration_batch_size(), + grace: default_expiration_grace(), } } } @@ -1215,6 +1239,36 @@ impl From } } +/// Runtime configuration for the event streaming. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct EventStreamingConfig { + /// Enable/disable event emission + #[serde(default)] + pub enabled: bool, + + /// Maximum number of events to buffer before applying backpressure. + #[serde(default = "default_event_queue_capacity")] + pub event_queue_capacity: NonZeroUsize, + + /// Kafka-specific configuration. + #[serde(default)] + pub kafka: Option, +} + +impl Default for EventStreamingConfig { + fn default() -> Self { + Self { + enabled: false, + event_queue_capacity: default_event_queue_capacity(), + kafka: None, + } + } +} + +pub fn default_event_queue_capacity() -> NonZeroUsize { + NonZeroUsize::new(1024).expect("default event queue capacity is non-zero") +} + #[cfg(test)] mod tests { use super::*; @@ -1677,4 +1731,146 @@ mod tests { ); } } + + /// Mirrors the `event_streaming_config` field on `Config`: `#[serde(default)]` + /// over `Option`. Used to exercise the present/absent + /// behavior without constructing a full `Config`. + #[derive(serde::Deserialize)] + struct EventStreamingWrapper { + #[serde(default)] + event_streaming_config: Option, + } + + /// When the section is absent entirely, the optional field stays `None` so + /// event streaming is simply off rather than failing to parse. + #[test] + fn event_streaming_config_absent_is_none() { + let wrapper: EventStreamingWrapper = + serde_json::from_str("{}").expect("deserialization failed"); + assert!( + wrapper.event_streaming_config.is_none(), + "absent event_streaming_config should deserialize to None" + ); + } + + /// A present-but-empty section falls back to every field default rather than + /// requiring operators to spell out keys they don't care about. + #[test] + fn event_streaming_config_present_empty_uses_defaults() { + let wrapper: EventStreamingWrapper = + serde_json::from_str(r#"{ "event_streaming_config": {} }"#) + .expect("deserialization failed"); + + let config = wrapper + .event_streaming_config + .expect("event_streaming_config should be Some"); + + assert!(!config.enabled, "enabled should default to false"); + assert_eq!( + config.event_queue_capacity, + default_event_queue_capacity(), + "event_queue_capacity should default to 1024" + ); + assert!(config.kafka.is_none(), "kafka should default to None"); + } + + /// A fully specified section round-trips every field, including the nested Kafka block. + #[test] + fn event_streaming_config_parses_full() { + let json = r#"{ + "enabled": true, + "event_queue_capacity": 2048, + "kafka": { + "brokers": ["broker-1:9092", "broker-2:9092"], + "topic": "custom.topic", + "partitions": 8, + "sasl_mechanism": "PLAIN", + "sasl_username": "user", + "sasl_password": "pass", + "tls_enabled": true, + "tls_ca_cert_path": "/etc/ssl/ca.pem" + } + }"#; + + let config: EventStreamingConfig = + serde_json::from_str(json).expect("deserialization failed"); + + assert!(config.enabled, "enabled mismatch"); + assert_eq!( + config.event_queue_capacity, + NonZeroUsize::new(2048).unwrap(), + "event_queue_capacity mismatch" + ); + + let kafka = config.kafka.expect("kafka should be Some"); + assert_eq!( + kafka.brokers, + vec!["broker-1:9092".to_string(), "broker-2:9092".to_string()], + "brokers mismatch" + ); + assert_eq!(kafka.topic, "custom.topic", "topic mismatch"); + assert_eq!(kafka.partitions, 8, "partitions mismatch"); + assert_eq!( + kafka.sasl_mechanism.as_deref(), + Some("PLAIN"), + "sasl_mechanism mismatch" + ); + assert_eq!( + kafka.sasl_username.as_deref(), + Some("user"), + "sasl_username mismatch" + ); + assert_eq!( + kafka.sasl_password.as_deref(), + Some("pass"), + "sasl_password mismatch" + ); + assert!(kafka.tls_enabled, "tls_enabled mismatch"); + assert_eq!( + kafka.tls_ca_cert_path.as_deref(), + Some(std::path::Path::new("/etc/ssl/ca.pem")), + "tls_ca_cert_path mismatch" + ); + } + + /// A Kafka block with only the required `brokers` falls back to the topic + /// and partition defaults and leaves the optional auth/TLS fields unset. + #[test] + fn event_streaming_config_kafka_minimal_uses_defaults() { + let json = r#"{ + "kafka": { + "brokers": ["broker:9092"] + } + }"#; + + let config: EventStreamingConfig = + serde_json::from_str(json).expect("deserialization failed"); + + let kafka = config.kafka.expect("kafka should be Some"); + assert_eq!( + kafka.topic, "dipper.subgraph.indexing.agreement.events", + "topic should fall back to default" + ); + assert_eq!( + kafka.partitions, 16, + "partitions should fall back to default" + ); + assert!( + kafka.sasl_mechanism.is_none(), + "sasl_mechanism should be None" + ); + assert!( + kafka.sasl_username.is_none(), + "sasl_username should be None" + ); + assert!( + kafka.sasl_password.is_none(), + "sasl_password should be None" + ); + assert!(!kafka.tls_enabled, "tls_enabled should default to false"); + assert!( + kafka.tls_ca_cert_path.is_none(), + "tls_ca_cert_path should be None" + ); + } } diff --git a/bin/dipper-service/src/indexer_rpc_client.rs b/bin/dipper-service/src/indexer_rpc_client.rs index 7810c5ba..be82bf3c 100644 --- a/bin/dipper-service/src/indexer_rpc_client.rs +++ b/bin/dipper-service/src/indexer_rpc_client.rs @@ -371,6 +371,7 @@ mod tests { subgraph_deployment_id: deployment_id, protocol_network: 42161, chain_id: 1, + proposed_at: 0, }, }; @@ -456,6 +457,7 @@ mod tests { .unwrap(), protocol_network: 42161, chain_id: 1, + proposed_at: 0, }, } } diff --git a/bin/dipper-service/src/main.rs b/bin/dipper-service/src/main.rs index 30680149..8052a097 100644 --- a/bin/dipper-service/src/main.rs +++ b/bin/dipper-service/src/main.rs @@ -2,6 +2,9 @@ use std::{env, path::PathBuf, sync::Arc}; use async_signal::{Signal, Signals}; use dipper_iisa::{self as iisa}; +use dipper_producer::events::{ + SubgraphIndexingAgreementEventsProducer, SubgraphIndexingAgreementsEventsEmitter, +}; use futures_lite::StreamExt; use thegraph_core::alloy::signers::local::PrivateKeySigner; use tokio::task::JoinSet; @@ -11,6 +14,7 @@ use self::{ config::DEFAULT_MAX_CANDIDATES, registry::RegistryProvider, signing::eip712::Eip712Signer, worker::queue::QueueImpl, }; +use crate::config::EventStreamingConfig; mod admin_rpc_server; mod cancel_dispatch; @@ -21,6 +25,8 @@ mod indexer_rpc_client; mod network; mod registry; mod signing; +#[cfg(test)] +mod test_support; mod worker; #[global_allocator] @@ -353,6 +359,11 @@ pub async fn main() -> anyhow::Result<()> { _ => None, }; + // - The Subgraph Indexing Agreements Events emitter + // Optional, enabled by the config + let subgraph_indexing_agreements_events_emitter = + create_subgraph_indexing_agreements_events_emitter(&conf.event_streaming_config).await; + // Shared notify: worker signals the chain_listener when proposals are // dispatched so it switches from 300s idle polling to 5s immediately. let chain_listener_notify = Arc::new(tokio::sync::Notify::new()); @@ -423,6 +434,8 @@ pub async fn main() -> anyhow::Result<()> { unresponsive_breaker, dips_accepting_cache, concurrency: conf.worker_concurrency, + subgraph_indexing_agreements_events_emitter: + subgraph_indexing_agreements_events_emitter.clone(), }; worker::service::new(ctx) }; @@ -487,6 +500,8 @@ pub async fn main() -> anyhow::Result<()> { agreement_conf: chain_listener_agreement_conf.clone(), config: chain_listener_conf.clone(), chain_listener_notify: chain_listener_notify.clone(), + subgraph_indexing_agreements_events_emitter: + subgraph_indexing_agreements_events_emitter.clone(), }; let (handle, service) = network::service::chain_listener::new(ctx); Some((handle, service)) @@ -543,6 +558,8 @@ pub async fn main() -> anyhow::Result<()> { max_candidates: DEFAULT_MAX_CANDIDATES, registry: registry.clone(), worker: worker_handle.queue().clone(), + subgraph_indexing_agreements_events_emitter: + subgraph_indexing_agreements_events_emitter.clone(), }; admin_rpc_server::service::new(config, ctx) @@ -673,6 +690,12 @@ pub async fn main() -> anyhow::Result<()> { indexer_urls_handle.stop().await; tracing::trace!("stopped indexer URLs service"); + // All event producers have stopped; flush buffered diagnostic events to + // the broker before exit (bounded by an internal timeout). + tracing::trace!("flushing lifecycle event stream"); + subgraph_indexing_agreements_events_emitter.flush().await; + tracing::trace!("flushed lifecycle event stream"); + tracing::trace!("shutting down DB connection pool"); db_conn.close().await; tracing::trace!("shut down DB connection pool"); @@ -711,6 +734,8 @@ enum SignalHandlerError { RegistrationFailed(std::io::Error), } +// -------- Private app helpers -------- + /// Signal handler for the application async fn signal_task() -> Result { let mut signals = Signals::new([Signal::Term, Signal::Int, Signal::Quit, Signal::Abort]) @@ -731,3 +756,35 @@ async fn signal_task() -> Result { // Fallthrough Ok(AppSignal::Shutdown) } + +/// Initializes a SubgraphIndexingAgreementsEventsEmitter instance based on the config, if enabled. +/// +/// Always returns a SubgraphIndexingAgreementsEventsEmitter instance +/// - if no kafka config, returns as disabled +/// - if enabled is false, returns as disabled +/// - if initialization fails, returns as disabled +async fn create_subgraph_indexing_agreements_events_emitter( + config: &Option, +) -> Arc { + let Some(event_streaming_conf) = &config else { + return Arc::new(SubgraphIndexingAgreementsEventsEmitter::disabled()); + }; + if !event_streaming_conf.enabled { + return Arc::new(SubgraphIndexingAgreementsEventsEmitter::disabled()); + } + + let Some(kafka_config) = &event_streaming_conf.kafka else { + tracing::warn!("Events enabled, but no Kafka config provided. Returning disabled"); + return Arc::new(SubgraphIndexingAgreementsEventsEmitter::disabled()); + }; + + // The broker is connected in the background (retrying if unreachable at + // startup), so a transient boot-time outage no longer disables events for the + // whole run, and startup is never blocked on broker availability. + tracing::info!("Subgraph Indexing Agreements Event streaming enabled"); + + Arc::new(SubgraphIndexingAgreementsEventsEmitter::enabled( + kafka_config.clone(), + event_streaming_conf.event_queue_capacity.get(), + )) +} diff --git a/bin/dipper-service/src/network/service/chain_events.rs b/bin/dipper-service/src/network/service/chain_events.rs index 1a483f43..a7fdd36d 100644 --- a/bin/dipper-service/src/network/service/chain_events.rs +++ b/bin/dipper-service/src/network/service/chain_events.rs @@ -118,6 +118,18 @@ pub struct AgreementStateSnapshot { /// Block number of the latest state change. Used as the pagination /// cursor for subsequent polls. pub last_state_change_block: u64, + /// Timestamp of when the agreement was accepted on-chain (`0` if never + /// accepted). Sourced from the subgraph's `acceptedAt`. + pub accepted_at: u64, + /// Transaction hash (0x hex) of the on-chain accept. 32-byte zero sentinel + /// if never accepted. Sourced from the subgraph's `acceptedAtTx`. + pub accepted_tx: String, + /// Timestamp of when the agreement was canceled (`0` if not canceled). + /// Sourced from the subgraph's `canceledAt`. + pub canceled_at: u64, + /// Transaction hash (0x hex) of the on-chain cancel. 32-byte zero sentinel + /// if not canceled. Sourced from the subgraph's `canceledAtTx`. + pub canceled_tx: String, } /// Errors that can occur when fetching snapshots. @@ -248,6 +260,18 @@ struct IndexingAgreementEntity { canceled_by: String, /// Block number as string (BigInt in GraphQL) last_state_change_block: String, + /// Accept timestamp as string (BigInt in GraphQL); `"0"` if never accepted. + #[serde(default)] + accepted_at: String, + /// Hex-encoded tx hash of the on-chain accept (32-byte zero if never accepted). + #[serde(default)] + accepted_at_tx: String, + /// Cancel timestamp as string (BigInt in GraphQL); `"0"` if not canceled. + #[serde(default)] + canceled_at: String, + /// Hex-encoded tx hash of the on-chain cancel (32-byte zero if not canceled). + #[serde(default)] + canceled_at_tx: String, } #[derive(Debug, Deserialize)] @@ -443,6 +467,10 @@ impl ChainEventSource for SubgraphEventSource { state canceledBy lastStateChangeBlock + acceptedAt + acceptedAtTx + canceledAt + canceledAtTx }} {meta_clause} {{ block {{ @@ -693,9 +721,38 @@ fn parse_snapshot(entity: &IndexingAgreementEntity) -> Option u64 { + match raw.parse::() { + Ok(value) => value, + Err(_) => { + if !raw.is_empty() { + tracing::warn!( + field, + raw, + agreement_id, + "unparseable BigInt timestamp in subgraph response; defaulting to 0" + ); + } + 0 + } + } +} + /// Parse the GraphQL state enum string into the Rust enum. fn parse_state(s: &str) -> Option { match s { @@ -955,6 +1012,12 @@ mod tests { state: "Accepted".to_string(), canceled_by: "0x".to_string(), last_state_change_block: "150".to_string(), + accepted_at: "1700000001".to_string(), + accepted_at_tx: "0x1111111111111111111111111111111111111111111111111111111111111111" + .to_string(), + canceled_at: "0".to_string(), + canceled_at_tx: "0x0000000000000000000000000000000000000000000000000000000000000000" + .to_string(), } } @@ -964,6 +1027,34 @@ mod tests { assert_eq!(snapshot.last_state_change_block, 150); assert_eq!(snapshot.state, AgreementState::Accepted); assert_eq!(snapshot.canceled_by, Address::ZERO); + // Phase 4 enrichment: audit fields parse from the entity. + assert_eq!(snapshot.accepted_at, 1_700_000_001); + assert_eq!( + snapshot.accepted_tx, + "0x1111111111111111111111111111111111111111111111111111111111111111" + ); + assert_eq!(snapshot.canceled_at, 0); + assert_eq!( + snapshot.canceled_tx, + "0x0000000000000000000000000000000000000000000000000000000000000000" + ); + } + + #[test] + fn parse_bigint_timestamp_handles_valid_unset_and_malformed() { + // Valid number parses. + assert_eq!( + parse_bigint_timestamp("1700000001", "acceptedAt", "0xabc"), + 1_700_000_001 + ); + // "0" / empty are the expected unset encodings -> 0 (and not flagged). + assert_eq!(parse_bigint_timestamp("0", "acceptedAt", "0xabc"), 0); + assert_eq!(parse_bigint_timestamp("", "acceptedAt", "0xabc"), 0); + // A genuinely malformed value defaults to 0 (and is logged as a warning). + assert_eq!( + parse_bigint_timestamp("not-a-number", "acceptedAt", "0xabc"), + 0 + ); } #[test] @@ -1109,6 +1200,10 @@ mod tests { state: AgreementState::Accepted, canceled_by: Address::ZERO, last_state_change_block: block, + accepted_at: 0, + accepted_tx: String::new(), + canceled_at: 0, + canceled_tx: String::new(), } } diff --git a/bin/dipper-service/src/network/service/chain_listener.rs b/bin/dipper-service/src/network/service/chain_listener.rs index a934be15..6bdcba19 100644 --- a/bin/dipper-service/src/network/service/chain_listener.rs +++ b/bin/dipper-service/src/network/service/chain_listener.rs @@ -30,9 +30,18 @@ //! indexer-initiated ones. `canceledBy` is still carried as the canceler //! address for observability. -use std::{future::Future, sync::Arc, time::Duration}; +use std::{ + future::Future, + sync::Arc, + time::{Duration, Instant}, +}; use dipper_core::ids::IndexingAgreementId; +// The concrete emitter is only constructed in tests (`disabled()`); the runtime +// paths depend on the `dyn` trait, so keep this import test-gated. +#[cfg(test)] +use dipper_producer::events::SubgraphIndexingAgreementsEventsEmitter; +use dipper_producer::{events::SubgraphIndexingAgreementEventsProducer, proto}; use thegraph_core::alloy::primitives::Address; use tokio::{ sync::{Notify, mpsc}, @@ -111,6 +120,9 @@ pub struct Ctx { /// Signalled by the worker when proposals are dispatched, waking the listener /// from the slow (300s) idle interval so it starts fast-polling immediately. pub chain_listener_notify: Arc, + /// Emits `accepted` and `terminated` lifecycle events as on-chain state is reconciled. + pub subgraph_indexing_agreements_events_emitter: + Arc, } /// State persisted in the database @@ -148,6 +160,7 @@ where agreement_conf, config, chain_listener_notify, + subgraph_indexing_agreements_events_emitter, } = ctx; let service = async move { @@ -211,6 +224,10 @@ where // Starts at SWEEP_POLLS so the first poll runs the sweep, // recovering any pre-startup orphans. let mut polls_since_sweep: u64 = SWEEP_POLLS; + // Pause the event sweeps after a Kafka send failure, backing off from one + // poll interval up to the idle interval, so a hung broker cannot stall the + // poll loop on every iteration. + let mut broker_cooldown = BrokerCooldown::new(fast_interval, slow_interval); // Wall-clock instant of the last successful chain-timestamp // persist. Used to bound how fast the persisted timestamp can // advance: a hostile response that stays just under the skew @@ -327,6 +344,41 @@ where } } + // Announce accepted / terminated / expired every poll, but pause the + // sweeps during a broker outage: each sweep makes one blocking send + // that can hang, so unchecked they would stall polling every iteration. + let now = Instant::now(); + if broker_cooldown.is_paused(now) { + tracing::debug!( + backoff_secs = broker_cooldown.backoff().as_secs(), + "event broker in cooldown; skipping event sweeps this poll" + ); + } else { + let health = run_event_sweeps( + ®istry, + &agreement_conf, + subgraph_indexing_agreements_events_emitter.as_ref(), + ) + .await; + match broker_cooldown.record(health, now) { + CooldownTransition::Entered => tracing::warn!( + backoff_secs = broker_cooldown.backoff().as_secs(), + "event broker send failed; pausing event sweeps until backoff elapses" + ), + CooldownTransition::Extended => tracing::warn!( + backoff_secs = broker_cooldown.backoff().as_secs(), + "event broker still unreachable; extending event sweep pause" + ), + CooldownTransition::Recovered => { + tracing::info!("event broker recovered; resuming event sweeps") + } + CooldownTransition::Unchanged => {} + } + } + + // The cancellation crash-recovery sweeps stay on the slower cadence: + // the steady-state fan-out fires from finalize on a fresh accept, so + // running these every poll is wasted DB work. polls_since_sweep += 1; if polls_since_sweep >= SWEEP_POLLS { sweep_executable_pending_cancellations(®istry, &chain_client, &agreement_conf) @@ -525,7 +577,7 @@ where } let agreement = agreements_by_id.remove(&snapshot.agreement_id); - match prepare_reconciliation(&snapshot, agreement, worker_queue).await { + match prepare_reconciliation(&snapshot, agreement, registry, worker_queue).await { Ok(Some(prep)) => prepared.push(prep), Ok(None) => {} Err(err) => { @@ -544,7 +596,7 @@ where // pre-fills every input id in `outcomes` (with `default()` // for rows whose CAS guard didn't match), so a missing id // is an unambiguous signal that the whole batch failed. - let items: Vec = prepared.iter().map(|p| p.item).collect(); + let items: Vec = prepared.iter().map(|p| p.item.clone()).collect(); let outcomes = if items.is_empty() { std::collections::HashMap::new() } else { @@ -696,6 +748,9 @@ where struct PreparedReconciliation { item: ReconciliationItem, agreement: IndexingAgreement, + /// Address that appears in the cancel log line. The accept/cancel audit is + /// carried on `item.audit` and persisted with the transition; the `accepted` + /// and `terminated` events are emitted later by their sweeps. indexer: Address, } @@ -738,12 +793,14 @@ fn apply_chain_ts_drift_cap( /// no DB write is needed. Caller pre-fetches the agreement (in batch for /// the chain_listener loop, single-row for tests) and passes `None` /// when the snapshot is for an agreement we don't track locally. -async fn prepare_reconciliation( +async fn prepare_reconciliation( snapshot: &AgreementStateSnapshot, agreement: Option, + registry: &R, worker_queue: &W, ) -> anyhow::Result> where + R: AgreementRegistry + Sync, W: WorkerQueue, { tracing::debug!( @@ -787,6 +844,22 @@ where // Background: on-chain cleanup of a rejected-then-accepted agreement. .cancel_rejected_agreement_on_chain(agreement.id, JobPriority::Background) .await?; + + // This row goes Rejected -> Canceled without ever transiting + // AcceptedOnChain, so `apply_reconciliation` never records the accept. + // Persist it from the snapshot so the eventual `terminated` (emitted by + // the sweep after the cancel) is eligible (`accepted_at IS NOT NULL`) and + // carries accurate accept data. + if let Err(err) = registry + .record_accepted_audit(&agreement.id, snapshot.accepted_at, &snapshot.accepted_tx) + .await + { + tracing::warn!( + agreement_id = %agreement.id, + error = %err, + "failed to record accepted audit for rejected-then-accepted agreement" + ); + } } // Both transitions are applied atomically downstream so the @@ -820,6 +893,16 @@ where agreement_id: agreement.id, apply_accept, cancel: cancel_kind, + // Capture the observed on-chain payload for the transition(s) + // this item applies, so it is persisted with the status change + // and a later emission sweep can rebuild the event from the row. + audit: crate::registry::ReconciliationAudit { + accepted_at: apply_accept.then_some(snapshot.accepted_at), + accepted_tx: apply_accept.then(|| snapshot.accepted_tx.clone()), + canceled_at: cancel_kind.map(|_| snapshot.canceled_at), + canceled_by: cancel_kind.map(|_| snapshot.canceled_by.to_string()), + canceled_tx: cancel_kind.map(|_| snapshot.canceled_tx.clone()), + }, }, agreement, indexer: snapshot.indexer, @@ -846,7 +929,7 @@ async fn finalize_reconciliation( config: &crate::config::IndexingAgreementConfig, ) -> anyhow::Result<()> where - R: AgreementRegistry + PendingCancellationRegistry, + R: AgreementRegistry + PendingCancellationRegistry + Sync, T: ChainClient, { if outcome.did_accept { @@ -862,6 +945,11 @@ where reason, "agreement state transition" ); + + // The `accepted` event is NOT emitted here. `apply_reconciliation` + // persisted the accept audit (accepted_at/tx) in the same transaction as + // this transition; `sweep_pending_accepted_events` is the single + // authoritative emitter and will announce it durably. } if outcome.did_cancel { @@ -877,6 +965,11 @@ where ), None => {} } + + // The `terminated` event is NOT emitted here. `apply_reconciliation` + // persisted the cancel audit (canceled_at/by/tx) in the same transaction + // as this transition; `sweep_pending_terminated_events` is the single + // authoritative emitter and will announce it durably. } // Gated on `did_accept` so it only fires on a fresh AcceptedOnChain @@ -903,14 +996,15 @@ async fn reconcile_agreement( config: &crate::config::IndexingAgreementConfig, ) -> anyhow::Result<()> where - R: AgreementRegistry + PendingCancellationRegistry, + R: AgreementRegistry + PendingCancellationRegistry + Sync, W: WorkerQueue, T: ChainClient, { let agreement = registry .get_indexing_agreement_by_id(&snapshot.agreement_id) .await?; - let Some(prep) = prepare_reconciliation(snapshot, agreement, worker_queue).await? else { + let Some(prep) = prepare_reconciliation(snapshot, agreement, registry, worker_queue).await? + else { return Ok(()); }; @@ -936,7 +1030,7 @@ async fn execute_pending_cancellations( config: &crate::config::IndexingAgreementConfig, ) -> anyhow::Result<()> where - R: AgreementRegistry + PendingCancellationRegistry, + R: AgreementRegistry + PendingCancellationRegistry + Sync, T: ChainClient, { let pending = registry @@ -967,6 +1061,7 @@ where Some(a) => a, }; + let mut on_chain_cancel_tx: Option = None; match crate::cancel_dispatch::cancel_agreement_on_chain( chain_client, &old_agreement, @@ -981,6 +1076,7 @@ where %tx_hash, "Submitted on-chain cancellation for replaced agreement" ); + on_chain_cancel_tx = Some(tx_hash.to_string()); } Ok(None) => { tracing::info!( @@ -1035,6 +1131,28 @@ where old_agreement_id = %cancellation.old_agreement_id, "Canceled old agreement on-chain and in dipper DB after replacement confirmed" ); + + // Record the cancel audit so `sweep_pending_terminated_events` can emit + // the `terminated` durably. Crucially, the sweep only emits for rows that + // were genuinely accepted on-chain (`accepted_at IS NOT NULL`): a + // proposed-but-never-accepted replacement records audit here but is never + // swept, so it produces no spurious `terminated`. + let manager = config.recurring_agreement_manager().to_string(); + if let Err(err) = registry + .record_cancel_audit( + &cancellation.old_agreement_id, + dipper_core::time::now_secs(), + &manager, + on_chain_cancel_tx.as_deref(), + ) + .await + { + tracing::warn!( + old_agreement_id = %cancellation.old_agreement_id, + error = %err, + "failed to record cancel audit; terminated event may emit with fallback fields" + ); + } } if transient_failures > 0 { @@ -1067,7 +1185,7 @@ async fn sweep_orphan_canceled_agreements( chain_client: &T, config: &crate::config::IndexingAgreementConfig, ) where - R: AgreementRegistry, + R: AgreementRegistry + Sync, T: ChainClient, { let orphans = match registry @@ -1094,6 +1212,7 @@ async fn sweep_orphan_canceled_agreements( ); for agreement in orphans { + let mut on_chain_cancel_tx: Option = None; match crate::cancel_dispatch::cancel_agreement_on_chain(chain_client, &agreement, config) .await { @@ -1103,6 +1222,7 @@ async fn sweep_orphan_canceled_agreements( %tx_hash, "Submitted on-chain cancel for orphan agreement" ); + on_chain_cancel_tx = Some(tx_hash.to_string()); } Ok(None) => { tracing::info!( @@ -1129,10 +1249,395 @@ async fn sweep_orphan_canceled_agreements( agreement_id = %agreement.id, "Failed to mark orphan agreement as canceled in local DB" ); + continue; + } + + // Orphan (previously accepted) agreement canceled on-chain by dipper. + // Record the cancel audit; `sweep_pending_terminated_events` emits the + // `terminated` durably (the row is `AcceptedOnChain` -> terminal, so it is + // sweep-eligible). + let manager = config.recurring_agreement_manager().to_string(); + if let Err(err) = registry + .record_cancel_audit( + &agreement.id, + dipper_core::time::now_secs(), + &manager, + on_chain_cancel_tx.as_deref(), + ) + .await + { + tracing::warn!( + agreement_id = %agreement.id, + error = %err, + "failed to record cancel audit; terminated event may emit with fallback fields" + ); } } } +/// Health of a durable event sweep, reported to the poll loop so it can pause +/// the sweeps during a Kafka outage instead of blocking on every poll. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SweepHealth { + /// At least one event was sent and the broker accepted it. + Sent, + /// A send failed; the broker is likely unreachable. + Failed, + /// Nothing was pending, so the broker's reachability is unknown. + Idle, +} + +impl SweepHealth { + /// Combine two outcomes: a failure dominates, then a send, then idle. + fn merge(self, other: SweepHealth) -> SweepHealth { + match (self, other) { + (SweepHealth::Failed, _) | (_, SweepHealth::Failed) => SweepHealth::Failed, + (SweepHealth::Sent, _) | (_, SweepHealth::Sent) => SweepHealth::Sent, + _ => SweepHealth::Idle, + } + } +} + +/// The cooldown change worth logging after folding in a sweep round. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CooldownTransition { + /// Broker went from healthy to paused. + Entered, + /// A retry failed again, so the backoff grew. + Extended, + /// First successful send after a pause. + Recovered, + /// Nothing worth logging. + Unchanged, +} + +/// Backoff clock that pauses the durable event sweeps after a Kafka send +/// failure. Each sweep makes one blocking send that can hang for the produce +/// timeout, so an outage would otherwise stall the chain poll loop every poll. +struct BrokerCooldown { + /// Initial backoff and the value reset to on recovery: one poll interval. + base: Duration, + /// Backoff ceiling, so a down broker is retried at least this often. + ceiling: Duration, + /// When the current pause began, or `None` while the broker is healthy. + down_since: Option, + /// Current backoff length. + backoff: Duration, +} + +impl BrokerCooldown { + fn new(base: Duration, ceiling: Duration) -> Self { + Self { + base, + ceiling, + down_since: None, + backoff: base, + } + } + + /// Whether the event sweeps should be skipped at `now`. + fn is_paused(&self, now: Instant) -> bool { + matches!(self.down_since, Some(since) if now.duration_since(since) < self.backoff) + } + + fn backoff(&self) -> Duration { + self.backoff + } + + /// Fold a sweep round's health into the clock, returning what to log. A send + /// clears the pause and resets the backoff; a failure enters or extends it + /// (doubling up to the ceiling); an idle round leaves the state untouched. + fn record(&mut self, health: SweepHealth, now: Instant) -> CooldownTransition { + match health { + SweepHealth::Sent => { + let recovered = self.down_since.take().is_some(); + self.backoff = self.base; + if recovered { + CooldownTransition::Recovered + } else { + CooldownTransition::Unchanged + } + } + SweepHealth::Failed if self.down_since.is_some() => { + self.backoff = self.backoff.saturating_mul(2).min(self.ceiling); + self.down_since = Some(now); + CooldownTransition::Extended + } + SweepHealth::Failed => { + self.backoff = self.base; + self.down_since = Some(now); + CooldownTransition::Entered + } + SweepHealth::Idle => CooldownTransition::Unchanged, + } + } +} + +/// Run the three durable event sweeps in order, stopping early if one reports a +/// broker send failure so the rest are not attempted this poll. Accepted before +/// terminated: one snapshot can accept then cancel, and `accepted` must lead. +async fn run_event_sweeps( + registry: &R, + config: &crate::config::IndexingAgreementConfig, + events_emitter: &dyn SubgraphIndexingAgreementEventsProducer, +) -> SweepHealth +where + R: AgreementRegistry + Sync, +{ + let accepted = sweep_pending_accepted_events(registry, events_emitter).await; + if accepted == SweepHealth::Failed { + return accepted; + } + let health = + accepted.merge(sweep_pending_terminated_events(registry, config, events_emitter).await); + if health == SweepHealth::Failed { + return health; + } + health.merge(sweep_pending_expired_events(registry, events_emitter).await) +} + +/// Emit `terminated` events durably for agreements that have reached a terminal +/// cancel state but not yet been announced. +/// +/// This is the single authoritative emitter for `terminated`: every cancel path +/// (indexer-observed on-chain, or dipper-initiated) marks the row terminal and +/// leaves `terminated_event_emitted_at` NULL; this sweep re-derives the event +/// from the row, sends it synchronously, and stamps the marker only after the +/// broker confirms. If the send fails or the process crashes first, the marker +/// stays NULL and the next sweep retries -- self-healing, at-least-once. +/// +/// Eligibility (`get_agreements_pending_terminated_emission`) requires +/// `accepted_at IS NOT NULL`, so a never-accepted agreement that was only ever +/// cancelled locally never produces a `terminated` -- there was nothing live +/// on-chain to terminate. +async fn sweep_pending_terminated_events( + registry: &R, + config: &crate::config::IndexingAgreementConfig, + events_emitter: &dyn SubgraphIndexingAgreementEventsProducer, +) -> SweepHealth +where + R: AgreementRegistry + Sync, +{ + let pending = match registry + .get_agreements_pending_terminated_emission(SWEEP_BATCH_SIZE) + .await + { + Ok(pending) => pending, + Err(err) => { + tracing::warn!(error = %err, "failed to query agreements pending terminated emission"); + return SweepHealth::Idle; + } + }; + + if pending.is_empty() { + return SweepHealth::Idle; + } + + tracing::debug!(count = pending.len(), "emitting pending terminated events"); + + for p in pending { + // Count runs after the row is already terminal, so it excludes this one. + let remaining = + crate::registry::remaining_accepted_indexing_agreements(registry, &p.deployment_id) + .await; + + // Fall back to the row's terminal-transition time and the manager address + // when the on-chain cancel audit was never recorded. + let terminated_at = p + .canceled_at + .unwrap_or_else(|| p.updated_at.unix_timestamp().max(0) as u64); + let terminated_by = p + .canceled_by + .clone() + .unwrap_or_else(|| config.recurring_agreement_manager().to_string()); + + let event = proto::SubgraphIndexingAgreementTerminated { + agreement_id: p.agreement_id.to_string(), + indexer: p.indexer_id.to_string(), + terminated_at, + terminated_by, + terminated_tx: p.canceled_tx.clone().unwrap_or_default(), + remaining_accepted_indexing_agreements: remaining, + }; + + match events_emitter + .emit_subgraph_indexing_agreement_terminated(p.deployment_id, p.protocol_network, event) + .await + { + Ok(()) => { + if let Err(err) = registry + .mark_terminated_event_emitted(&p.agreement_id) + .await + { + // Sent but not stamped: the next sweep re-sends (deduped by + // consumers on agreement_id). Better than dropping. + tracing::warn!( + agreement_id = %p.agreement_id, + error = %err, + "emitted terminated event but failed to stamp marker; may re-emit" + ); + } + } + Err(err) => { + // A send failure almost always means the broker is unreachable, + // so the rest of this batch would fail too. Stop now and retry the + // whole batch next sweep -- avoids a burst of failed sends and log + // spam during an outage. Nothing is lost (markers stay unset). + tracing::warn!( + agreement_id = %p.agreement_id, + error = %err, + "failed to emit terminated event; pausing batch, will retry next sweep" + ); + return SweepHealth::Failed; + } + } + } + SweepHealth::Sent +} + +/// Emit `accepted` events durably for agreements that reached accepted on-chain +/// but haven't been announced. Mirror of [`sweep_pending_terminated_events`]: +/// re-derive from the row, send synchronously, stamp the marker only on a +/// confirmed send. +/// +/// Eligibility (`get_agreements_pending_accepted_emission`) requires +/// `accepted_at IS NOT NULL` (so pre-feature rows are never backfilled) but is +/// NOT gated on current status: an agreement accepted and then cancelled in a +/// single snapshot is already terminal yet must still emit its `accepted` (which +/// is why this sweep runs before the terminated sweep). +async fn sweep_pending_accepted_events( + registry: &R, + events_emitter: &dyn SubgraphIndexingAgreementEventsProducer, +) -> SweepHealth +where + R: AgreementRegistry + Sync, +{ + let pending = match registry + .get_agreements_pending_accepted_emission(SWEEP_BATCH_SIZE) + .await + { + Ok(pending) => pending, + Err(err) => { + tracing::warn!(error = %err, "failed to query agreements pending accepted emission"); + return SweepHealth::Idle; + } + }; + + if pending.is_empty() { + return SweepHealth::Idle; + } + + tracing::debug!(count = pending.len(), "emitting pending accepted events"); + + for p in pending { + let event = proto::SubgraphIndexingAgreementAccepted { + agreement_id: p.agreement_id.to_string(), + indexer: p.indexer_id.to_string(), + accepted_at: p.accepted_at, + accepted_tx: p.accepted_tx.clone(), + ends_at: p.ends_at, + payer: p.payer.to_string(), + }; + + match events_emitter + .emit_subgraph_indexing_agreement_accepted(p.deployment_id, p.protocol_network, event) + .await + { + Ok(()) => { + if let Err(err) = registry.mark_accepted_event_emitted(&p.agreement_id).await { + tracing::warn!( + agreement_id = %p.agreement_id, + error = %err, + "emitted accepted event but failed to stamp marker; may re-emit" + ); + } + } + Err(err) => { + // Broker likely unreachable; the rest of the batch would fail too. + // Stop and retry next sweep (markers stay unset -> nothing lost). + tracing::warn!( + agreement_id = %p.agreement_id, + error = %err, + "failed to emit accepted event; pausing batch, will retry next sweep" + ); + return SweepHealth::Failed; + } + } + } + SweepHealth::Sent +} + +/// Emit `request.expired` events durably for agreements marked `Expired` but not +/// yet announced. Mirror of the accepted/terminated sweeps. +/// +/// Running here (after the drain, which recovers `Expired -> AcceptedOnChain` on +/// a late on-chain accept) means a row that was prematurely marked `Expired` is +/// no longer selected once recovered -- so a premature `expired` never +/// contradicts a subsequent `accepted`. +async fn sweep_pending_expired_events( + registry: &R, + events_emitter: &dyn SubgraphIndexingAgreementEventsProducer, +) -> SweepHealth +where + R: AgreementRegistry + Sync, +{ + let pending = match registry + .get_agreements_pending_expired_emission(SWEEP_BATCH_SIZE) + .await + { + Ok(pending) => pending, + Err(err) => { + tracing::warn!(error = %err, "failed to query agreements pending expired emission"); + return SweepHealth::Idle; + } + }; + + if pending.is_empty() { + return SweepHealth::Idle; + } + + tracing::debug!(count = pending.len(), "emitting pending expired events"); + + for p in pending { + let event = proto::SubgraphIndexingAgreementRequestExpired { + agreement_id: p.agreement_id.to_string(), + indexer: p.indexer_id.to_string(), + request_proposed_at: p.request_proposed_at, + request_expired_at: p.request_expired_at, + }; + + match events_emitter + .emit_subgraph_indexing_agreement_request_expired( + p.deployment_id, + p.protocol_network, + event, + ) + .await + { + Ok(()) => { + if let Err(err) = registry.mark_expired_event_emitted(&p.agreement_id).await { + tracing::warn!( + agreement_id = %p.agreement_id, + error = %err, + "emitted expired event but failed to stamp marker; may re-emit" + ); + } + } + Err(err) => { + // Broker likely unreachable; the rest of the batch would fail too. + // Stop and retry next sweep (markers stay unset -> nothing lost). + tracing::warn!( + agreement_id = %p.agreement_id, + error = %err, + "failed to emit expired event; pausing batch, will retry next sweep" + ); + return SweepHealth::Failed; + } + } + } + SweepHealth::Sent +} + /// Recover stranded pending cancellations from a partial-progress crash. /// /// `execute_pending_cancellations` is invoked from `reconcile_agreement` @@ -1152,7 +1657,7 @@ async fn sweep_executable_pending_cancellations( chain_client: &T, config: &crate::config::IndexingAgreementConfig, ) where - R: AgreementRegistry + PendingCancellationRegistry, + R: AgreementRegistry + PendingCancellationRegistry + Sync, T: ChainClient, { let targets = match registry @@ -1244,6 +1749,7 @@ mod tests { .unwrap(), protocol_network: ChainId::from(42161u64), chain_id: ChainId::from(1u64), + proposed_at: 0, }, } } @@ -1274,6 +1780,15 @@ mod tests { }) } + // Deterministic audit values used by `make_snapshot` so emit tests can assert + // the enriched `accepted`/`terminated` tx + timestamp fields. + const TEST_ACCEPTED_AT: u64 = 1_700_000_001; + const TEST_ACCEPTED_TX: &str = + "0x1111111111111111111111111111111111111111111111111111111111111111"; + const TEST_CANCELED_AT: u64 = 1_700_000_002; + const TEST_CANCELED_TX: &str = + "0x2222222222222222222222222222222222222222222222222222222222222222"; + fn make_snapshot( agreement_id: IndexingAgreementId, state: AgreementState, @@ -1287,6 +1802,10 @@ mod tests { state, canceled_by, last_state_change_block: 100, + accepted_at: TEST_ACCEPTED_AT, + accepted_tx: TEST_ACCEPTED_TX.to_string(), + canceled_at: TEST_CANCELED_AT, + canceled_tx: TEST_CANCELED_TX.to_string(), } } @@ -1309,6 +1828,9 @@ mod tests { marked_accepted_on_chain: Vec, marked_canceled_by_requester: Vec, marked_canceled_by_indexer: Vec, + /// Ids passed to `record_cancel_audit` -- the signal a cancel path drives + /// the terminated event (the sweep emits from this audit). + recorded_cancel_audit: Vec, pending_cancellations: std::collections::HashMap< IndexingAgreementId, Vec, @@ -1319,6 +1841,7 @@ mod tests { chain_listener_state_updates: Vec<(Cursor, Option)>, initial_last_processed_block: u64, fail_batch: bool, + pending_accepted: Vec, } impl MockRegistry { @@ -1408,6 +1931,14 @@ mod tests { .contains(id) } + fn was_cancel_audit_recorded(&self, id: &IndexingAgreementId) -> bool { + self.state + .lock() + .unwrap() + .recorded_cancel_audit + .contains(id) + } + fn was_pending_cancellation_deleted( &self, new_id: &IndexingAgreementId, @@ -1435,6 +1966,10 @@ mod tests { fn set_fail_batch(&self, fail: bool) { self.state.lock().unwrap().fail_batch = fail; } + + fn seed_pending_accepted(&self, event: crate::registry::PendingAcceptedEvent) { + self.state.lock().unwrap().pending_accepted.push(event); + } } #[async_trait::async_trait] @@ -1498,6 +2033,13 @@ mod tests { Ok(vec![]) } + async fn count_accepted_agreements_by_deployment( + &self, + _deployment_id: &DeploymentId, + ) -> RegistryResult { + Ok(0) + } + async fn register_new_indexing_agreement( &self, _params: crate::registry::NewAgreementParams, @@ -1544,6 +2086,21 @@ mod tests { Ok(()) } + async fn record_cancel_audit( + &self, + agreement_id: &IndexingAgreementId, + _canceled_at: u64, + _canceled_by: &str, + _canceled_tx: Option<&str>, + ) -> RegistryResult<()> { + self.state + .lock() + .unwrap() + .recorded_cancel_audit + .push(*agreement_id); + Ok(()) + } + async fn apply_reconciliation( &self, id: &IndexingAgreementId, @@ -1750,6 +2307,13 @@ mod tests { async fn get_agreement_fee_rates(&self) -> RegistryResult> { Ok(vec![]) } + + async fn get_agreements_pending_accepted_emission( + &self, + _limit: i64, + ) -> RegistryResult> { + Ok(self.state.lock().unwrap().pending_accepted.clone()) + } } #[async_trait::async_trait] @@ -1985,6 +2549,78 @@ mod tests { assert!(!worker_queue.was_cancellation_queued(&agreement_id)); } + #[tokio::test] + async fn reconcile_accept_marks_accepted_without_emitting() { + use crate::test_support::CapturingEventsProducer; + + let registry = MockRegistry::new(); + let chain_client = MockChainClient::default(); + let worker_queue = MockWorkerQueue::default(); + let events = CapturingEventsProducer::new(); + let agreement_id = IndexingAgreementId::from_bytes(rand::random()); + registry.add_agreement(agreement_id, IndexingAgreementStatus::Created); + + let snapshot = make_snapshot(agreement_id, AgreementState::Accepted, Address::ZERO); + reconcile_agreement( + &snapshot, + ®istry, + &worker_queue, + &chain_client, + test_agreement_conf().as_ref(), + ) + .await + .expect("reconcile ok"); + + // reconcile flips the row to AcceptedOnChain (and, in production, persists + // the accept audit in the same transaction). It no longer emits + // `accepted` -- `sweep_pending_accepted_events` is the sole emitter. + assert!(registry.was_marked_accepted_on_chain(&agreement_id)); + assert!( + events.events().is_empty(), + "reconcile emits no accepted event; the sweep does" + ); + } + + #[tokio::test] + async fn reconcile_indexer_cancel_marks_canceled_without_emitting() { + use crate::test_support::CapturingEventsProducer; + + let registry = MockRegistry::new(); + let chain_client = MockChainClient::default(); + let worker_queue = MockWorkerQueue::default(); + let events = CapturingEventsProducer::new(); + let agreement_id = IndexingAgreementId::from_bytes(rand::random()); + registry.add_agreement(agreement_id, IndexingAgreementStatus::AcceptedOnChain); + + // Indexer-initiated on-chain cancel: canceled_by != signer (ZERO) -> ByIndexer. + let indexer: Address = "0x1234567890123456789012345678901234567890" + .parse() + .unwrap(); + let snapshot = make_snapshot( + agreement_id, + AgreementState::CanceledByServiceProvider, + indexer, + ); + reconcile_agreement( + &snapshot, + ®istry, + &worker_queue, + &chain_client, + test_agreement_conf().as_ref(), + ) + .await + .expect("reconcile ok"); + + // reconcile flips the row to CanceledByIndexer (and, in production, + // persists the cancel audit in the same transaction). It no longer emits + // `terminated` -- `sweep_pending_terminated_events` is the sole emitter. + assert!(registry.was_marked_canceled_by_indexer(&agreement_id)); + assert!( + events.events().is_empty(), + "reconcile emits no terminated event; the sweep does" + ); + } + #[tokio::test] async fn test_reconcile_queues_cancellation_for_rejected() { let registry = MockRegistry::new(); @@ -2340,6 +2976,62 @@ mod tests { assert!(remaining.is_empty()); } + #[tokio::test] + async fn test_pending_cancellations_records_cancel_audit() { + // execute_pending no longer emits `terminated` directly; it records the + // cancel audit and the chain_listener sweep announces it durably. + let registry = MockRegistry::new(); + let chain_client = MockChainClient::default(); + let new_id = IndexingAgreementId::from_bytes(rand::random()); + let old_id = IndexingAgreementId::from_bytes(rand::random()); + + registry.add_agreement(new_id, IndexingAgreementStatus::AcceptedOnChain); + registry.add_agreement(old_id, IndexingAgreementStatus::AcceptedOnChain); + registry.add_pending_cancellation(new_id, old_id); + + let result = execute_pending_cancellations( + &new_id, + ®istry, + &chain_client, + test_agreement_conf().as_ref(), + ) + .await; + + assert!(result.is_ok()); + assert!(registry.was_marked_canceled_by_requester(&old_id)); + assert!( + registry.was_cancel_audit_recorded(&old_id), + "cancel audit recorded so the sweep can emit `terminated`" + ); + } + + #[tokio::test] + async fn test_pending_cancellations_failed_cancel_records_no_audit() { + let registry = MockRegistry::new(); + let chain_client = MockChainClient::default(); + let new_id = IndexingAgreementId::from_bytes(rand::random()); + let old_fail = IndexingAgreementId::from_bytes(rand::random()); + + registry.add_agreement(new_id, IndexingAgreementStatus::AcceptedOnChain); + registry.add_agreement(old_fail, IndexingAgreementStatus::AcceptedOnChain); + registry.add_pending_cancellation(new_id, old_fail); + registry.fail_cancel_for(old_fail); + + let result = execute_pending_cancellations( + &new_id, + ®istry, + &chain_client, + test_agreement_conf().as_ref(), + ) + .await; + + // The chain cancel failed, the row is NOT marked canceled, and no cancel + // audit is recorded (so the sweep emits nothing). + assert!(result.is_err()); + assert!(!registry.was_marked_canceled_by_requester(&old_fail)); + assert!(!registry.was_cancel_audit_recorded(&old_fail)); + } + #[tokio::test] async fn test_pending_cancellations_transient_failure_retains_record() { let registry = MockRegistry::new(); @@ -2665,6 +3357,9 @@ mod tests { config, agreement_conf: test_agreement_conf(), chain_listener_notify: Arc::new(tokio::sync::Notify::new()), + subgraph_indexing_agreements_events_emitter: Arc::new( + SubgraphIndexingAgreementsEventsEmitter::disabled(), + ), }; let (handle, service) = new(ctx); @@ -2717,6 +3412,10 @@ mod tests { state: super::super::chain_events::AgreementState::Accepted, canceled_by: Address::ZERO, last_state_change_block: ((i + 1) as u64) * 10, + accepted_at: 0, + accepted_tx: String::new(), + canceled_at: 0, + canceled_tx: String::new(), }]); } @@ -2742,6 +3441,9 @@ mod tests { config, agreement_conf: test_agreement_conf(), chain_listener_notify: Arc::new(tokio::sync::Notify::new()), + subgraph_indexing_agreements_events_emitter: Arc::new( + SubgraphIndexingAgreementsEventsEmitter::disabled(), + ), }; let (handle, service) = new(ctx); @@ -2791,6 +3493,10 @@ mod tests { state: super::super::chain_events::AgreementState::Accepted, canceled_by: Address::ZERO, last_state_change_block: 50, + accepted_at: 0, + accepted_tx: String::new(), + canceled_at: 0, + canceled_tx: String::new(), }]); } @@ -2816,6 +3522,9 @@ mod tests { config, agreement_conf: test_agreement_conf(), chain_listener_notify: Arc::new(tokio::sync::Notify::new()), + subgraph_indexing_agreements_events_emitter: Arc::new( + SubgraphIndexingAgreementsEventsEmitter::disabled(), + ), }; let (handle, service) = new(ctx); @@ -2865,6 +3574,10 @@ mod tests { state: super::super::chain_events::AgreementState::Accepted, canceled_by: Address::ZERO, last_state_change_block: ((i + 1) as u64) * 10, + accepted_at: 0, + accepted_tx: String::new(), + canceled_at: 0, + canceled_tx: String::new(), }]); } @@ -3134,6 +3847,9 @@ mod tests { config, agreement_conf: test_agreement_conf(), chain_listener_notify: notify.clone(), + subgraph_indexing_agreements_events_emitter: Arc::new( + SubgraphIndexingAgreementsEventsEmitter::disabled(), + ), }; let (handle, service) = new(ctx); @@ -3183,6 +3899,29 @@ mod tests { ); } + #[tokio::test] + async fn test_orphan_sweep_records_cancel_audit() { + // The orphan sweep no longer emits `terminated` directly; it records the + // cancel audit and `sweep_pending_terminated_events` announces it. + let registry = MockRegistry::new(); + let chain_client = MockChainClient::default(); + let agreement_id = IndexingAgreementId::from_bytes(rand::random()); + let request_id = IndexingRequestId::new(); + + registry.add_agreement(agreement_id, IndexingAgreementStatus::AcceptedOnChain); + registry.set_agreement_request_id(agreement_id, request_id); + registry.mark_request_canceled(request_id); + + sweep_orphan_canceled_agreements(®istry, &chain_client, test_agreement_conf().as_ref()) + .await; + + assert!(registry.was_marked_canceled_by_requester(&agreement_id)); + assert!( + registry.was_cancel_audit_recorded(&agreement_id), + "cancel audit recorded so the sweep can emit `terminated`" + ); + } + #[tokio::test] async fn test_orphan_sweep_handles_already_canceled_on_chain() { // Idempotency check: the chain reports the agreement is already @@ -3242,4 +3981,230 @@ mod tests { // were registered. assert!(chain_client.cancels.lock().unwrap().is_empty()); } + + // -- broker cooldown and sweep-health tests -- + + /// An emitter whose durable sends always fail, standing in for a hung or + /// unreachable Kafka broker so a sweep reports `SweepHealth::Failed`. + struct FailingEmitter; + + #[async_trait::async_trait] + impl SubgraphIndexingAgreementEventsProducer for FailingEmitter { + fn produce_subgraph_indexing_agreement_request_received( + &self, + _: DeploymentId, + _: ChainId, + _: proto::SubgraphIndexingAgreementRequestReceived, + ) { + } + fn produce_subgraph_indexing_agreement_proposed( + &self, + _: DeploymentId, + _: ChainId, + _: proto::SubgraphIndexingAgreementProposed, + ) { + } + fn produce_subgraph_indexing_agreement_n_indexers_unavailable( + &self, + _: DeploymentId, + _: ChainId, + _: proto::SubgraphIndexingAgreementNIndexersUnavailable, + ) { + } + async fn emit_subgraph_indexing_agreement_accepted( + &self, + _: DeploymentId, + _: ChainId, + _: proto::SubgraphIndexingAgreementAccepted, + ) -> Result<(), dipper_producer::events::EmitError> { + Err(dipper_producer::events::EmitError::Send( + "broker down".to_string(), + )) + } + async fn emit_subgraph_indexing_agreement_request_expired( + &self, + _: DeploymentId, + _: ChainId, + _: proto::SubgraphIndexingAgreementRequestExpired, + ) -> Result<(), dipper_producer::events::EmitError> { + Err(dipper_producer::events::EmitError::Send( + "broker down".to_string(), + )) + } + async fn emit_subgraph_indexing_agreement_terminated( + &self, + _: DeploymentId, + _: ChainId, + _: proto::SubgraphIndexingAgreementTerminated, + ) -> Result<(), dipper_producer::events::EmitError> { + Err(dipper_producer::events::EmitError::Send( + "broker down".to_string(), + )) + } + } + + fn sample_pending_accepted() -> crate::registry::PendingAcceptedEvent { + crate::registry::PendingAcceptedEvent { + agreement_id: IndexingAgreementId::from_bytes(rand::random()), + indexer_id: "0x1234567890123456789012345678901234567890" + .parse() + .unwrap(), + deployment_id: "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG" + .parse() + .unwrap(), + protocol_network: ChainId::from(42161u64), + accepted_at: 100, + accepted_tx: "0xabc".to_string(), + ends_at: 200, + payer: Address::ZERO, + } + } + + #[test] + fn broker_cooldown_healthy_broker_never_pauses() { + let base = Duration::from_secs(30); + let mut cd = BrokerCooldown::new(base, Duration::from_secs(300)); + let now = Instant::now(); + assert!(!cd.is_paused(now)); + assert_eq!( + cd.record(SweepHealth::Idle, now), + CooldownTransition::Unchanged + ); + assert_eq!( + cd.record(SweepHealth::Sent, now), + CooldownTransition::Unchanged + ); + assert!(!cd.is_paused(now)); + } + + #[test] + fn broker_cooldown_enters_and_pauses_for_one_interval() { + let base = Duration::from_secs(30); + let mut cd = BrokerCooldown::new(base, Duration::from_secs(300)); + let t0 = Instant::now(); + assert_eq!( + cd.record(SweepHealth::Failed, t0), + CooldownTransition::Entered + ); + assert_eq!(cd.backoff(), base); + assert!(cd.is_paused(t0)); + assert!(cd.is_paused(t0 + base - Duration::from_millis(1))); + // At exactly one interval the pause is over and the sweeps retry. + assert!(!cd.is_paused(t0 + base)); + } + + #[test] + fn broker_cooldown_doubles_backoff_up_to_ceiling() { + let base = Duration::from_secs(30); + let ceiling = Duration::from_secs(300); + let mut cd = BrokerCooldown::new(base, ceiling); + let mut t = Instant::now(); + assert_eq!( + cd.record(SweepHealth::Failed, t), + CooldownTransition::Entered + ); + for secs in [60u64, 120, 240, 300, 300] { + t += Duration::from_secs(1); + assert_eq!( + cd.record(SweepHealth::Failed, t), + CooldownTransition::Extended + ); + assert_eq!(cd.backoff(), Duration::from_secs(secs)); + } + } + + #[test] + fn broker_cooldown_recovers_and_resets_backoff() { + let base = Duration::from_secs(30); + let mut cd = BrokerCooldown::new(base, Duration::from_secs(300)); + let t0 = Instant::now(); + cd.record(SweepHealth::Failed, t0); + cd.record(SweepHealth::Failed, t0 + Duration::from_secs(1)); + assert_eq!(cd.backoff(), Duration::from_secs(60)); + let t1 = t0 + Duration::from_secs(120); + assert_eq!( + cd.record(SweepHealth::Sent, t1), + CooldownTransition::Recovered + ); + assert!(!cd.is_paused(t1)); + assert_eq!(cd.backoff(), base, "recovery resets the backoff"); + assert_eq!( + cd.record(SweepHealth::Sent, t1), + CooldownTransition::Unchanged, + "a later healthy sweep is not another recovery" + ); + } + + #[test] + fn broker_cooldown_idle_round_leaves_pause_intact() { + let base = Duration::from_secs(30); + let mut cd = BrokerCooldown::new(base, Duration::from_secs(300)); + let t0 = Instant::now(); + cd.record(SweepHealth::Failed, t0); + assert_eq!( + cd.record(SweepHealth::Idle, t0 + base), + CooldownTransition::Unchanged + ); + assert_eq!(cd.backoff(), base); + assert_eq!( + cd.record(SweepHealth::Failed, t0 + base), + CooldownTransition::Extended, + "still down, so a fresh failure extends rather than re-enters" + ); + } + + #[test] + fn sweep_health_merge_prefers_failure_then_sent() { + use SweepHealth::{Failed, Idle, Sent}; + assert_eq!(Failed.merge(Sent), Failed); + assert_eq!(Sent.merge(Failed), Failed); + assert_eq!(Idle.merge(Failed), Failed); + assert_eq!(Sent.merge(Idle), Sent); + assert_eq!(Idle.merge(Sent), Sent); + assert_eq!(Idle.merge(Idle), Idle); + } + + #[tokio::test] + async fn accepted_sweep_reports_sent_when_broker_accepts() { + let registry = MockRegistry::new(); + registry.seed_pending_accepted(sample_pending_accepted()); + let events = crate::test_support::CapturingEventsProducer::new(); + + let health = sweep_pending_accepted_events(®istry, &events).await; + + assert_eq!(health, SweepHealth::Sent); + assert_eq!( + events.events().len(), + 1, + "the pending accepted row is announced" + ); + } + + #[tokio::test] + async fn accepted_sweep_reports_failed_when_broker_rejects() { + let registry = MockRegistry::new(); + registry.seed_pending_accepted(sample_pending_accepted()); + + let health = sweep_pending_accepted_events(®istry, &FailingEmitter).await; + + assert_eq!( + health, + SweepHealth::Failed, + "a failed send must trip the cooldown" + ); + } + + #[tokio::test] + async fn event_sweeps_report_idle_when_nothing_pending() { + let registry = MockRegistry::new(); + let events = crate::test_support::CapturingEventsProducer::new(); + + let health = run_event_sweeps(®istry, test_agreement_conf().as_ref(), &events).await; + + assert_eq!(health, SweepHealth::Idle); + assert!( + events.events().is_empty(), + "nothing pending means nothing announced" + ); + } } diff --git a/bin/dipper-service/src/network/service/expiration.rs b/bin/dipper-service/src/network/service/expiration.rs index e16b69cb..d79f2dec 100644 --- a/bin/dipper-service/src/network/service/expiration.rs +++ b/bin/dipper-service/src/network/service/expiration.rs @@ -1,4 +1,5 @@ //! Expires `Created` agreements whose RCA deadline has passed on-chain. +//! //! Compares deadlines against the chain_listener's block timestamp, not wall //! clock time. Stays dormant when no chain time is available. @@ -118,10 +119,19 @@ where dormant_cycles = 0; + // Hold expiration back by the configured grace period: only agreements + // whose deadline is at least `grace` seconds behind chain time are + // considered expired. This lets the chain_listener sync a + // within-deadline on-chain accept (flipping the row to + // `AcceptedOnChain`) before we mark it `Expired`, so a lagging local + // row can't produce a premature `expired` that contradicts a later + // `accepted`. + let expiry_threshold = chain_ts.saturating_sub(config.grace.as_secs()); + // Query expired agreements using chain time (with timeout) let query_result = tokio::time::timeout( DB_QUERY_TIMEOUT, - registry.get_expired_created_agreements(config.batch_size, chain_ts), + registry.get_expired_created_agreements(config.batch_size, expiry_threshold), ) .await; @@ -175,6 +185,13 @@ where new_status = "Expired", "agreement state transition" ); + + // The `request.expired` event is NOT emitted here. The row + // is now `Expired`; `sweep_pending_expired_events` on the + // chain_listener tick announces it durably -- and skips any + // row recovered to `AcceptedOnChain` in the meantime, so a + // premature expiry never contradicts a later `accepted`. + // Clean up pending cancellations: the replacement expired // before on-chain acceptance, so old agreements stay active. if let Err(err) = registry @@ -303,16 +320,18 @@ mod tests { use dipper_core::ids::{IndexingAgreementId, IndexingRequestId}; use thegraph_core::{ DeploymentId, IndexerId, - alloy::primitives::{Address, ChainId}, + alloy::primitives::{Address, ChainId, U256}, }; + use time::OffsetDateTime; use url::Url; use super::*; use crate::{ network::service::chain_listener::ChainListenerState, registry::{ - IndexingAgreement, IndexingRequest, PendingCancellation, Result as RegistryResult, - StubAgreementRegistry, + Indexer, IndexingAgreement, IndexingAgreementStatus, IndexingAgreementTerms, + IndexingAgreementTermsMetadata, IndexingRequest, IndexingRequestStatus, + PendingCancellation, Result as RegistryResult, StubAgreementRegistry, }, worker::{queue::JobId, service::WorkerQueue}, }; @@ -337,6 +356,7 @@ mod tests { expired_agreements: Vec, marked_expired: Vec, get_expired_calls: Vec, + indexing_request: Option, } impl MockExpirationRegistry { @@ -347,6 +367,7 @@ mod tests { expired_agreements: vec![], marked_expired: vec![], get_expired_calls: vec![], + indexing_request: None, })), } } @@ -355,6 +376,18 @@ mod tests { self.state.lock().unwrap().chain_state = state; } + /// Agreements returned by the next `get_expired_created_agreements` call + /// (the mock drains them, so they are returned exactly once). + fn set_expired_agreements(&self, agreements: Vec) { + self.state.lock().unwrap().expired_agreements = agreements; + } + + /// The request returned by `get_indexing_request_by_id`. The loop fetches + /// it after marking an agreement expired (to queue reassessment). + fn set_indexing_request(&self, request: IndexingRequest) { + self.state.lock().unwrap().indexing_request = Some(request); + } + fn get_expired_calls(&self) -> Vec { self.state.lock().unwrap().get_expired_calls.clone() } @@ -429,7 +462,7 @@ mod tests { &self, _id: &IndexingRequestId, ) -> RegistryResult> { - unimplemented!() + Ok(self.state.lock().unwrap().indexing_request.clone()) } async fn get_indexing_requests_by_deployment_id( &self, @@ -521,6 +554,207 @@ mod tests { } } + // -- Helpers for the event-emission tests -- + + /// Build a `Created` agreement that the mock registry will report as expired. + /// Callers control the fields the `request.expired` event payload is derived + /// from: the deployment, protocol network, indexer, and `created_at`. + fn make_expired_agreement( + deployment: DeploymentId, + protocol_network: ChainId, + indexer_id: IndexerId, + created_at: OffsetDateTime, + ) -> IndexingAgreement { + IndexingAgreement { + id: IndexingAgreementId::from_bytes(rand::random()), + nonce_uuid: uuid::Uuid::now_v7(), + created_at, + updated_at: created_at, + status: IndexingAgreementStatus::Created, + indexing_request_id: IndexingRequestId::new(), + indexer: Indexer { + id: indexer_id, + url: "https://indexer.example.com".parse().unwrap(), + }, + terms: IndexingAgreementTerms { + payer: Address::ZERO, + service_provider: Address::ZERO, + data_service: Address::ZERO, + deadline: 0, + ends_at: 0, + max_initial_tokens: U256::ZERO, + max_ongoing_tokens_per_second: U256::ZERO, + min_seconds_per_collection: 0, + max_seconds_per_collection: 0, + conditions: 0, + metadata: IndexingAgreementTermsMetadata { + tokens_per_second: U256::ZERO, + tokens_per_entity_per_second: U256::ZERO, + subgraph_deployment_id: deployment, + protocol_network, + chain_id: 1u64, + proposed_at: 0, + }, + }, + last_block_height: None, + last_progress_at: None, + rejection_reason: None, + terms_version_hash: Some(vec![0u8; 32]), + } + } + + fn make_indexing_request(id: IndexingRequestId, deployment: DeploymentId) -> IndexingRequest { + IndexingRequest { + id, + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + status: IndexingRequestStatus::Open, + requested_by: Address::ZERO, + deployment_id: deployment, + deployment_chain_id: 1u64, + num_candidates: 1, + } + } + + #[tokio::test] + async fn test_marks_expired_agreement_once() { + // The expiration service marks the agreement `Expired`; the chain_listener + // sweep emits `request.expired` from the row. Assert the mark happened + // exactly once. (The `request_proposed_at`/`request_expired_at` derivation + // moved to `PendingExpiredEvent::from_row` and is covered there.) + const CHAIN_TS: u64 = 1_700_000_000; + let deployment: DeploymentId = "QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9" + .parse() + .unwrap(); + let protocol_network: ChainId = 42161; + let indexer_id = IndexerId::from(Address::from([0x11u8; 20])); + let created_at = OffsetDateTime::from_unix_timestamp(1_699_000_000).unwrap(); + + let registry = MockExpirationRegistry::new(); + registry.set_chain_state(Some(ChainListenerState { + _chain_id: 1337, + last_processed_block: 500, + last_processed_id: None, + last_processed_block_timestamp: Some(CHAIN_TS), + })); + let agreement = + make_expired_agreement(deployment, protocol_network, indexer_id, created_at); + let agr_id = agreement.id; + registry.set_indexing_request(make_indexing_request( + agreement.indexing_request_id, + deployment, + )); + registry.set_expired_agreements(vec![agreement]); + + let ctx = Ctx { + registry: registry.clone(), + worker_queue: MockWorkerQueue, + config: ExpirationConfig { + enabled: true, + interval: Duration::from_millis(10), + batch_size: 100, + grace: Duration::from_secs(0), + }, + chain_id: Some(1337), + }; + + let (handle, service) = new(ctx); + let svc = tokio::spawn(service); + + // Wait until the cycle has run (the agreement is drained exactly once), + // then stop. Subsequent cycles see an empty expired set. + tokio::time::sleep(Duration::from_millis(50)).await; + handle.stop().await; + svc.await.unwrap().unwrap(); + + assert_eq!( + registry.marked_expired(), + vec![agr_id], + "exactly one agreement marked Expired" + ); + } + + #[tokio::test] + async fn test_grace_holds_back_expiry_below_the_margin() { + // Item 8 guard: with a grace margin, agreements whose deadline is within + // `grace` of chain time are NOT yet queried as expired -- the effective + // threshold passed to the registry is `chain_ts - grace`. + const CHAIN_TS: u64 = 1_700_000_000; + const GRACE: u64 = 300; + + let registry = MockExpirationRegistry::new(); + registry.set_chain_state(Some(ChainListenerState { + _chain_id: 1337, + last_processed_block: 500, + last_processed_id: None, + last_processed_block_timestamp: Some(CHAIN_TS), + })); + + let ctx = Ctx { + registry: registry.clone(), + worker_queue: MockWorkerQueue, + config: ExpirationConfig { + enabled: true, + interval: Duration::from_millis(10), + batch_size: 100, + grace: Duration::from_secs(GRACE), + }, + chain_id: Some(1337), + }; + + let (handle, service) = new(ctx); + let svc = tokio::spawn(service); + tokio::time::sleep(Duration::from_millis(50)).await; + handle.stop().await; + svc.await.unwrap().unwrap(); + + // The registry was queried with the grace-adjusted threshold, not raw chain time. + assert!( + registry.get_expired_calls().contains(&(CHAIN_TS - GRACE)), + "expiry threshold must be chain_ts - grace; calls: {:?}", + registry.get_expired_calls() + ); + } + + #[tokio::test] + async fn test_marks_nothing_when_nothing_expires() { + let registry = MockExpirationRegistry::new(); + // Chain time is available (so the service is active), but no agreement + // is configured as expired: get_expired_created_agreements returns empty. + registry.set_chain_state(Some(ChainListenerState { + _chain_id: 1337, + last_processed_block: 500, + last_processed_id: None, + last_processed_block_timestamp: Some(1_700_000_000), + })); + + let ctx = Ctx { + registry: registry.clone(), + worker_queue: MockWorkerQueue, + config: ExpirationConfig { + enabled: true, + interval: Duration::from_millis(10), + batch_size: 100, + grace: Duration::from_secs(0), + }, + chain_id: Some(1337), + }; + + let (handle, service) = new(ctx); + let svc = tokio::spawn(service); + + tokio::time::sleep(Duration::from_millis(50)).await; + handle.stop().await; + svc.await.unwrap().unwrap(); + + // The query ran (service was active), but nothing was marked expired. + assert!(!registry.get_expired_calls().is_empty()); + assert!( + registry.marked_expired().is_empty(), + "nothing should be marked expired when the expired set is empty" + ); + } + #[tokio::test] async fn test_skips_when_no_chain_id() { let registry = MockExpirationRegistry::new(); @@ -532,6 +766,7 @@ mod tests { enabled: true, interval: Duration::from_millis(10), batch_size: 100, + grace: Duration::from_secs(0), }, chain_id: None, }; @@ -559,6 +794,7 @@ mod tests { enabled: true, interval: Duration::from_millis(10), batch_size: 100, + grace: Duration::from_secs(0), }, chain_id: Some(1337), }; @@ -593,6 +829,7 @@ mod tests { enabled: true, interval: Duration::from_millis(10), batch_size: 100, + grace: Duration::from_secs(0), }, chain_id: Some(1337), }; @@ -625,6 +862,7 @@ mod tests { enabled: true, interval: Duration::from_millis(10), batch_size: 100, + grace: Duration::from_secs(0), }, chain_id: Some(1337), }; diff --git a/bin/dipper-service/src/network/service/indexer_urls.rs b/bin/dipper-service/src/network/service/indexer_urls.rs index 27be2433..371f9d01 100644 --- a/bin/dipper-service/src/network/service/indexer_urls.rs +++ b/bin/dipper-service/src/network/service/indexer_urls.rs @@ -40,6 +40,19 @@ impl Handle { self.rx_snapshot.borrow().get(id).cloned() } + /// Build a `Handle` directly from a seeded snapshot for tests. The stop + /// channel's receiver is dropped immediately; there is no background + /// service, the handle only serves `get_indexer_url` reads. + #[cfg(test)] + pub(crate) fn for_test(snapshot: Snapshot) -> Self { + let (tx_stop, _rx_stop) = mpsc::channel(1); + let (_tx_snapshot, rx_snapshot) = watch::channel(snapshot); + Self { + rx_snapshot, + tx_stop, + } + } + /// Signal the service to stop and wait for it to shut down; returns /// immediately if it is already stopped. pub async fn stop(&self) { diff --git a/bin/dipper-service/src/network/service/liveness_checker.rs b/bin/dipper-service/src/network/service/liveness_checker.rs index a7abd112..2d554e9f 100644 --- a/bin/dipper-service/src/network/service/liveness_checker.rs +++ b/bin/dipper-service/src/network/service/liveness_checker.rs @@ -476,6 +476,7 @@ async fn record_progress( /// /// If the on-chain cancel fails, the DB is not updated and reassessment is not /// queued, leaving the agreement in `AcceptedOnChain` for the next cycle to retry. +#[allow(clippy::too_many_arguments)] async fn cancel_and_reassess( agreement: &IndexingAgreement, registry: &R, @@ -490,6 +491,7 @@ async fn cancel_and_reassess( C: ChainClient + Send + Sync, { // 1. Cancel on-chain (mode-aware dispatch) + let mut on_chain_cancel_tx: Option = None; match crate::cancel_dispatch::cancel_agreement_on_chain(chain_client, agreement, agreement_conf) .await { @@ -499,6 +501,7 @@ async fn cancel_and_reassess( tx_hash = %tx_hash, "canceled stale agreement on-chain" ); + on_chain_cancel_tx = Some(tx_hash.to_string()); } Ok(None) => { tracing::info!( @@ -560,6 +563,27 @@ async fn cancel_and_reassess( } }; + // The accepted agreement was canceled on-chain because the indexer went + // stale. Record the cancel audit so the chain_listener's `terminated` sweep + // announces it durably: this row is marked `AbandonedByIndexer` (terminal) + // and was accepted on-chain, so it is sweep-eligible. + let manager = agreement_conf.recurring_agreement_manager().to_string(); + if let Err(err) = registry + .record_cancel_audit( + &agreement.id, + dipper_core::time::now_secs(), + &manager, + on_chain_cancel_tx.as_deref(), + ) + .await + { + tracing::warn!( + agreement_id = %agreement.id, + error = %err, + "failed to record cancel audit; terminated event may emit with fallback fields" + ); + } + // Clean up pending cancellations: if this abandoned agreement was a // replacement, the old agreement it was replacing should stay active. if let Err(err) = registry @@ -834,6 +858,7 @@ mod tests { subgraph_deployment_id: deployment_id, protocol_network: 1u64, chain_id: 1u64, + proposed_at: 0, }, }; IndexingAgreement { @@ -880,6 +905,9 @@ mod tests { abandoned: Arc>>, reassessments: Arc>>, chain_cancels: Arc>>, + /// Ids passed to `record_cancel_audit` -- the signal the handler drives + /// the terminated event (the chain_listener sweep emits from this audit). + cancel_audits: Arc>>, } struct MockRegistry { @@ -944,6 +972,17 @@ mod tests { .expect("mark_abandoned called more than once") } + async fn record_cancel_audit( + &self, + agreement_id: &IndexingAgreementId, + _canceled_at: u64, + _canceled_by: &str, + _canceled_tx: Option<&str>, + ) -> RegistryResult<()> { + self.calls.cancel_audits.lock().unwrap().push(*agreement_id); + Ok(()) + } + async fn get_agreement_fee_rates(&self) -> RegistryResult> { Ok(vec![]) } @@ -1461,6 +1500,49 @@ mod tests { assert_eq!(calls.reassessments.lock().unwrap().as_slice(), &[req_id]); } + #[tokio::test] + async fn cancel_and_reassess_records_cancel_audit() { + // The stale-agreement cancel no longer emits `terminated` directly: it + // records the cancel audit and the chain_listener sweep announces it. + let dep: DeploymentId = "QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9" + .parse() + .unwrap(); + let indexer_id = IndexerId::from(Address::ZERO); + let url: Url = "http://indexer.example.com/".parse().unwrap(); + let agreement = make_agreement( + indexer_id, + url, + dep, + Some(100), + Some(OffsetDateTime::now_utc()), + ); + let agr_id = agreement.id; + + let calls = MockCalls::default(); + let registry = MockRegistry::new(calls.clone(), agreement.clone()); + let queue = MockWorkerQueue { + calls: calls.clone(), + }; + let chain = MockChainClient::success(calls.clone()); + + cancel_and_reassess( + &agreement, + ®istry, + &queue, + &chain, + &test_agreement_conf(), + DB_TIMEOUT, + QUEUE_TIMEOUT, + ) + .await; + + assert_eq!( + calls.cancel_audits.lock().unwrap().as_slice(), + &[agr_id], + "exactly one cancel audit recorded for the stale agreement" + ); + } + #[tokio::test] async fn test_cancel_and_reassess_config_error_proceeds() { // Arrange: chain client disabled (ConfigError) → still mark abandoned and reassess diff --git a/bin/dipper-service/src/registry.rs b/bin/dipper-service/src/registry.rs index 281ad9bb..0509cb78 100644 --- a/bin/dipper-service/src/registry.rs +++ b/bin/dipper-service/src/registry.rs @@ -17,15 +17,16 @@ use thegraph_core::{ // Re-export for tests only #[cfg(test)] -pub use self::agreement::Indexer; +pub use self::agreement::{Indexer, PendingAcceptedEvent}; #[cfg(test)] pub use self::agreement_stub::StubAgreementRegistry; use self::result::Result as RegistryResult; pub use self::{ agreement::{ AgreementFeeRate, AgreementRegistry, CancelKind, IndexingAgreement, NewAgreementParams, - ReconciliationItem, ReconciliationOutcome, Status as IndexingAgreementStatus, - Terms as IndexingAgreementTerms, TermsMetadata as IndexingAgreementTermsMetadata, + ReconciliationAudit, ReconciliationItem, ReconciliationOutcome, + Status as IndexingAgreementStatus, Terms as IndexingAgreementTerms, + TermsMetadata as IndexingAgreementTermsMetadata, }, indexer_denylist::IndexerDenylistRegistry, indexing_request::{ @@ -50,6 +51,40 @@ impl From for dipper_pgregistry::NewAgreementParams { } } +/// Resolve the `remaining_accepted_indexing_agreements` value for a `terminated` +/// lifecycle event: the number of still-accepted agreements on the deployment. +/// +/// MUST be called AFTER the just-cancelled agreement's terminal status is persisted, +/// so the count (which matches only `AcceptedOnChain`) excludes it and returns only +/// the *other* agreements still active on the subgraph. +/// +/// Returns `-1` (unknown) on a registry error rather than a misleading `0` — the +/// proto defines `0` as "the Subgraph has no remaining accepted agreements", so a +/// transient DB failure must not be reported as "fully unstaffed". +pub(crate) async fn remaining_accepted_indexing_agreements( + registry: &R, + deployment_id: &DeploymentId, +) -> i32 +where + R: AgreementRegistry, +{ + match registry + .count_accepted_agreements_by_deployment(deployment_id) + .await + { + Ok(count) => i32::try_from(count).unwrap_or(i32::MAX), + Err(err) => { + tracing::warn!( + error = %err, + deployment_id = %deployment_id, + "failed to count remaining accepted agreements for terminated event; \ + emitting -1 (unknown)" + ); + -1 + } + } +} + /// Filter and log conversion errors instead of silently dropping them: use as /// `.filter_map(filter_map_with_logging)` so failed conversions leave a warning. fn filter_map_with_logging( @@ -152,6 +187,17 @@ impl IndexingRequestRegistry for RegistryProvider { .filter_map(filter_map_with_logging) .collect()) } + + async fn set_indexing_request_shortfall_active( + &self, + id: &IndexingRequestId, + active: bool, + ) -> RegistryResult { + Ok(self + .inner + .set_indexing_request_shortfall_active(id, active) + .await?) + } } #[async_trait] @@ -279,6 +325,15 @@ impl AgreementRegistry for RegistryProvider { .filter_map(filter_map_with_logging) .collect()) } + async fn count_accepted_agreements_by_deployment( + &self, + deployment_id: &DeploymentId, + ) -> RegistryResult { + self.inner + .count_accepted_agreements_by_deployment(deployment_id) + .await + .map_err(Into::into) + } async fn register_new_indexing_agreement( &self, params: NewAgreementParams, @@ -368,6 +423,13 @@ impl AgreementRegistry for RegistryProvider { } agreement::CancelKind::ByIndexer => dipper_pgregistry::CancelKind::ByIndexer, }), + audit: dipper_pgregistry::ReconciliationAudit { + accepted_at: item.audit.accepted_at, + accepted_tx: item.audit.accepted_tx.clone(), + canceled_at: item.audit.canceled_at, + canceled_by: item.audit.canceled_by.clone(), + canceled_tx: item.audit.canceled_tx.clone(), + }, }) .collect(); let outcomes = self.inner.apply_reconciliation_batch(&pg_items).await?; @@ -385,6 +447,121 @@ impl AgreementRegistry for RegistryProvider { .collect()) } + async fn get_agreements_pending_terminated_emission( + &self, + limit: i64, + ) -> RegistryResult> { + Ok(self + .inner + .get_agreements_pending_terminated_emission(limit) + .await? + .into_iter() + .map(|p| agreement::PendingTerminatedEvent { + agreement_id: p.agreement_id, + indexer_id: p.indexer_id, + deployment_id: p.deployment_id, + protocol_network: p.protocol_network, + canceled_at: p.canceled_at, + canceled_by: p.canceled_by, + canceled_tx: p.canceled_tx, + updated_at: p.updated_at, + }) + .collect()) + } + + async fn mark_terminated_event_emitted( + &self, + agreement_id: &IndexingAgreementId, + ) -> RegistryResult<()> { + self.inner + .mark_terminated_event_emitted(agreement_id) + .await?; + Ok(()) + } + + async fn get_agreements_pending_accepted_emission( + &self, + limit: i64, + ) -> RegistryResult> { + Ok(self + .inner + .get_agreements_pending_accepted_emission(limit) + .await? + .into_iter() + .map(|p| agreement::PendingAcceptedEvent { + agreement_id: p.agreement_id, + indexer_id: p.indexer_id, + deployment_id: p.deployment_id, + protocol_network: p.protocol_network, + accepted_at: p.accepted_at, + accepted_tx: p.accepted_tx, + ends_at: p.ends_at, + payer: p.payer, + }) + .collect()) + } + + async fn mark_accepted_event_emitted( + &self, + agreement_id: &IndexingAgreementId, + ) -> RegistryResult<()> { + self.inner.mark_accepted_event_emitted(agreement_id).await?; + Ok(()) + } + + async fn get_agreements_pending_expired_emission( + &self, + limit: i64, + ) -> RegistryResult> { + Ok(self + .inner + .get_agreements_pending_expired_emission(limit) + .await? + .into_iter() + .map(|p| agreement::PendingExpiredEvent { + agreement_id: p.agreement_id, + indexer_id: p.indexer_id, + deployment_id: p.deployment_id, + protocol_network: p.protocol_network, + request_proposed_at: p.request_proposed_at, + request_expired_at: p.request_expired_at, + }) + .collect()) + } + + async fn mark_expired_event_emitted( + &self, + agreement_id: &IndexingAgreementId, + ) -> RegistryResult<()> { + self.inner.mark_expired_event_emitted(agreement_id).await?; + Ok(()) + } + + async fn record_accepted_audit( + &self, + agreement_id: &IndexingAgreementId, + accepted_at: u64, + accepted_tx: &str, + ) -> RegistryResult<()> { + self.inner + .record_accepted_audit(agreement_id, accepted_at, accepted_tx) + .await?; + Ok(()) + } + + async fn record_cancel_audit( + &self, + agreement_id: &IndexingAgreementId, + canceled_at: u64, + canceled_by: &str, + canceled_tx: Option<&str>, + ) -> RegistryResult<()> { + self.inner + .record_cancel_audit(agreement_id, canceled_at, canceled_by, canceled_tx) + .await?; + Ok(()) + } + async fn get_expired_created_agreements( &self, batch_size: i64, diff --git a/bin/dipper-service/src/registry/agreement.rs b/bin/dipper-service/src/registry/agreement.rs index 8b4fdaf9..8248203f 100644 --- a/bin/dipper-service/src/registry/agreement.rs +++ b/bin/dipper-service/src/registry/agreement.rs @@ -51,11 +51,69 @@ pub struct ReconciliationOutcome { } /// One row's reconciliation request inside a batched apply. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct ReconciliationItem { pub agreement_id: IndexingAgreementId, pub apply_accept: bool, pub cancel: Option, + /// On-chain payload observed for this transition, persisted alongside the + /// status change so a later emission sweep can rebuild the lifecycle event + /// from the row alone. Only the fields relevant to the applied transition + /// are populated. + pub audit: ReconciliationAudit, +} + +/// Audit payload captured at reconcile time for later event emission. +/// +/// Timestamps are chain seconds; tx hashes and `canceled_by` are the 0x hex +/// string form matching the event wire representation. +#[derive(Debug, Clone, Default)] +pub struct ReconciliationAudit { + pub accepted_at: Option, + pub accepted_tx: Option, + pub canceled_at: Option, + pub canceled_by: Option, + pub canceled_tx: Option, +} + +/// A row that needs a `terminated` lifecycle event emitted, rebuilt from the +/// agreement row alone by the emission sweep. +#[derive(Debug, Clone)] +pub struct PendingTerminatedEvent { + pub agreement_id: IndexingAgreementId, + pub indexer_id: IndexerId, + pub deployment_id: DeploymentId, + pub protocol_network: ChainId, + pub canceled_at: Option, + pub canceled_by: Option, + pub canceled_tx: Option, + pub updated_at: OffsetDateTime, +} + +/// A row that needs an `accepted` lifecycle event emitted, rebuilt from the +/// agreement row alone by the emission sweep. +#[derive(Debug, Clone)] +pub struct PendingAcceptedEvent { + pub agreement_id: IndexingAgreementId, + pub indexer_id: IndexerId, + pub deployment_id: DeploymentId, + pub protocol_network: ChainId, + pub accepted_at: u64, + pub accepted_tx: String, + pub ends_at: u64, + pub payer: Address, +} + +/// A row that needs a `request.expired` lifecycle event emitted, rebuilt from +/// the agreement row alone by the emission sweep. +#[derive(Debug, Clone)] +pub struct PendingExpiredEvent { + pub agreement_id: IndexingAgreementId, + pub indexer_id: IndexerId, + pub deployment_id: DeploymentId, + pub protocol_network: ChainId, + pub request_proposed_at: u64, + pub request_expired_at: u64, } #[async_trait] @@ -148,6 +206,16 @@ pub trait AgreementRegistry { request_id: &IndexingRequestId, ) -> RegistryResult>; + /// Count the agreements for a deployment that are accepted on-chain. + /// + /// Populates `remaining_accepted_indexing_agreements` on the `terminated` + /// lifecycle event: the number of still-active accepted agreements left on + /// the Subgraph (deployment) after a cancellation. + async fn count_accepted_agreements_by_deployment( + &self, + deployment_id: &DeploymentId, + ) -> RegistryResult; + /// Register a new indexing agreement. /// /// The caller generates the `agreement_id` (on-chain bytes16) and @@ -243,6 +311,88 @@ pub trait AgreementRegistry { Ok(outcomes) } + /// Fetch agreements awaiting a `terminated` event: terminal-cancel status, + /// genuinely accepted on-chain, marker unset. Default returns empty so mocks + /// need not override; the production registry runs the real query. + async fn get_agreements_pending_terminated_emission( + &self, + _limit: i64, + ) -> RegistryResult> { + Ok(Vec::new()) + } + + /// Stamp the `terminated` emission marker after a confirmed broker send. + /// Default no-op so mocks need not override. + async fn mark_terminated_event_emitted( + &self, + _agreement_id: &IndexingAgreementId, + ) -> RegistryResult<()> { + Ok(()) + } + + /// Fetch agreements awaiting an `accepted` event: `AcceptedOnChain`, accept + /// observed by the new reconcile path, marker unset. Default returns empty so + /// mocks need not override; the production registry runs the real query. + async fn get_agreements_pending_accepted_emission( + &self, + _limit: i64, + ) -> RegistryResult> { + Ok(Vec::new()) + } + + /// Stamp the `accepted` emission marker after a confirmed broker send. + /// Default no-op so mocks need not override. + async fn mark_accepted_event_emitted( + &self, + _agreement_id: &IndexingAgreementId, + ) -> RegistryResult<()> { + Ok(()) + } + + /// Fetch agreements awaiting a `request.expired` event: `Expired`, marker + /// unset. Default returns empty so mocks need not override; the production + /// registry runs the real query. + async fn get_agreements_pending_expired_emission( + &self, + _limit: i64, + ) -> RegistryResult> { + Ok(Vec::new()) + } + + /// Stamp the `request.expired` emission marker after a confirmed broker send. + /// Default no-op so mocks need not override. + async fn mark_expired_event_emitted( + &self, + _agreement_id: &IndexingAgreementId, + ) -> RegistryResult<()> { + Ok(()) + } + + /// Record the accepted audit payload out-of-band (rejected-then-accepted + /// anomaly), marking the row as genuinely accepted so its eventual + /// `terminated` is sweep-eligible. Default no-op so mocks need not override. + async fn record_accepted_audit( + &self, + _agreement_id: &IndexingAgreementId, + _accepted_at: u64, + _accepted_tx: &str, + ) -> RegistryResult<()> { + Ok(()) + } + + /// Record the cancel audit payload for a dipper-initiated cancel so the + /// emission sweep can populate the `terminated` event fields. Default no-op + /// so mocks need not override. + async fn record_cancel_audit( + &self, + _agreement_id: &IndexingAgreementId, + _canceled_at: u64, + _canceled_by: &str, + _canceled_tx: Option<&str>, + ) -> RegistryResult<()> { + Ok(()) + } + /// Get `Created` agreements whose deadline has passed (by block timestamp). async fn get_expired_created_agreements( &self, @@ -495,6 +645,9 @@ pub struct TermsMetadata { pub protocol_network: ChainId, /// Indexed chain, e.g., `eip155:1` (Ethereum Mainnet). pub chain_id: ChainId, + + /// Unix-seconds proposal time, in the same clock as the acceptance deadline. + pub proposed_at: u64, } /// The status of the [`IndexingAgreement`]. @@ -642,6 +795,7 @@ impl From for TermsMetadata { subgraph_deployment_id: value.subgraph_deployment_id, protocol_network: value.protocol_network, chain_id: value.chain_id, + proposed_at: value.proposed_at, } } } @@ -672,6 +826,7 @@ impl From for dipper_pgregistry::IndexingAgreementTermsMetadata { subgraph_deployment_id: value.subgraph_deployment_id, protocol_network: value.protocol_network, chain_id: value.chain_id, + proposed_at: value.proposed_at, } } } @@ -722,6 +877,7 @@ pub mod fake_impl { subgraph_deployment_id: DeploymentId::dummy_with_rng(config, rng), protocol_network: ChainId::dummy_with_rng(config, rng), chain_id: ChainId::dummy_with_rng(config, rng), + proposed_at: u64::dummy_with_rng(config, rng), } } } diff --git a/bin/dipper-service/src/registry/agreement_stub.rs b/bin/dipper-service/src/registry/agreement_stub.rs index 79458d58..0b34dcaf 100644 --- a/bin/dipper-service/src/registry/agreement_stub.rs +++ b/bin/dipper-service/src/registry/agreement_stub.rs @@ -13,7 +13,8 @@ use thegraph_core::{ use super::{ agreement::{ AgreementFeeRate, AgreementRegistry, CancelKind, IndexingAgreement, NewAgreementParams, - ReconciliationItem, ReconciliationOutcome, + PendingAcceptedEvent, PendingExpiredEvent, PendingTerminatedEvent, ReconciliationItem, + ReconciliationOutcome, }, result::Result, }; @@ -232,6 +233,70 @@ pub trait StubAgreementRegistry: Send + Sync { async fn get_agreement_fee_rates(&self) -> Result> { unimplemented!("get_agreement_fee_rates") } + + async fn count_accepted_agreements_by_deployment( + &self, + _deployment_id: &DeploymentId, + ) -> Result { + unimplemented!("count_accepted_agreements_by_deployment") + } + + // The emission-sweep and audit methods mirror the real trait's defaults + // (empty batch / no-op) so only emission-focused mocks override them. + async fn get_agreements_pending_terminated_emission( + &self, + _limit: i64, + ) -> Result> { + Ok(Vec::new()) + } + + async fn mark_terminated_event_emitted( + &self, + _agreement_id: &IndexingAgreementId, + ) -> Result<()> { + Ok(()) + } + + async fn get_agreements_pending_accepted_emission( + &self, + _limit: i64, + ) -> Result> { + Ok(Vec::new()) + } + + async fn mark_accepted_event_emitted(&self, _agreement_id: &IndexingAgreementId) -> Result<()> { + Ok(()) + } + + async fn get_agreements_pending_expired_emission( + &self, + _limit: i64, + ) -> Result> { + Ok(Vec::new()) + } + + async fn mark_expired_event_emitted(&self, _agreement_id: &IndexingAgreementId) -> Result<()> { + Ok(()) + } + + async fn record_accepted_audit( + &self, + _agreement_id: &IndexingAgreementId, + _accepted_at: u64, + _accepted_tx: &str, + ) -> Result<()> { + Ok(()) + } + + async fn record_cancel_audit( + &self, + _agreement_id: &IndexingAgreementId, + _canceled_at: u64, + _canceled_by: &str, + _canceled_tx: Option<&str>, + ) -> Result<()> { + Ok(()) + } } // Every stub is a full AgreementRegistry: each method delegates to the stub @@ -445,4 +510,74 @@ impl AgreementRegistry for T { async fn get_agreement_fee_rates(&self) -> Result> { StubAgreementRegistry::get_agreement_fee_rates(self).await } + + async fn count_accepted_agreements_by_deployment( + &self, + deployment_id: &DeploymentId, + ) -> Result { + StubAgreementRegistry::count_accepted_agreements_by_deployment(self, deployment_id).await + } + + async fn get_agreements_pending_terminated_emission( + &self, + limit: i64, + ) -> Result> { + StubAgreementRegistry::get_agreements_pending_terminated_emission(self, limit).await + } + + async fn mark_terminated_event_emitted( + &self, + agreement_id: &IndexingAgreementId, + ) -> Result<()> { + StubAgreementRegistry::mark_terminated_event_emitted(self, agreement_id).await + } + + async fn get_agreements_pending_accepted_emission( + &self, + limit: i64, + ) -> Result> { + StubAgreementRegistry::get_agreements_pending_accepted_emission(self, limit).await + } + + async fn mark_accepted_event_emitted(&self, agreement_id: &IndexingAgreementId) -> Result<()> { + StubAgreementRegistry::mark_accepted_event_emitted(self, agreement_id).await + } + + async fn get_agreements_pending_expired_emission( + &self, + limit: i64, + ) -> Result> { + StubAgreementRegistry::get_agreements_pending_expired_emission(self, limit).await + } + + async fn mark_expired_event_emitted(&self, agreement_id: &IndexingAgreementId) -> Result<()> { + StubAgreementRegistry::mark_expired_event_emitted(self, agreement_id).await + } + + async fn record_accepted_audit( + &self, + agreement_id: &IndexingAgreementId, + accepted_at: u64, + accepted_tx: &str, + ) -> Result<()> { + StubAgreementRegistry::record_accepted_audit(self, agreement_id, accepted_at, accepted_tx) + .await + } + + async fn record_cancel_audit( + &self, + agreement_id: &IndexingAgreementId, + canceled_at: u64, + canceled_by: &str, + canceled_tx: Option<&str>, + ) -> Result<()> { + StubAgreementRegistry::record_cancel_audit( + self, + agreement_id, + canceled_at, + canceled_by, + canceled_tx, + ) + .await + } } diff --git a/bin/dipper-service/src/registry/indexing_request.rs b/bin/dipper-service/src/registry/indexing_request.rs index 3defe549..adc17fcb 100644 --- a/bin/dipper-service/src/registry/indexing_request.rs +++ b/bin/dipper-service/src/registry/indexing_request.rs @@ -60,6 +60,23 @@ pub trait IndexingRequestRegistry { min_age_seconds: i64, batch_size: i64, ) -> RegistryResult>; + + /// Atomically set the request's coverage-shortfall latch, returning whether + /// the value actually changed (a single guarded UPDATE, so the transition is + /// race-free). + /// + /// Drives the `n_indexers_unavailable` signal: reassessment fires it once on + /// the transition INTO shortfall (`Ok(true)` with `active == true`) and clears + /// the latch on recovery (`Ok(true)` with `active == false`), rather than + /// re-emitting on every recurring reassessment of a persistently short request. + /// Default `Ok(false)` (never transitions) so mocks need not override. + async fn set_indexing_request_shortfall_active( + &self, + _id: &IndexingRequestId, + _active: bool, + ) -> RegistryResult { + Ok(false) + } } /// What `set_indexing_target_candidates` actually did. diff --git a/bin/dipper-service/src/test_support.rs b/bin/dipper-service/src/test_support.rs new file mode 100644 index 00000000..4ef6eb37 --- /dev/null +++ b/bin/dipper-service/src/test_support.rs @@ -0,0 +1,159 @@ +//! Shared test utilities for the dipper-service crate. +//! +//! Currently provides [`CapturingEventsProducer`], a test double for +//! [`SubgraphIndexingAgreementEventsProducer`] that records every produced event +//! so handler/service tests can assert exactly which lifecycle events were emitted +//! (and with which payloads) without a Kafka backend. + +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use dipper_producer::{ + events::{EmitError, SubgraphIndexingAgreementEventsProducer}, + proto, +}; +use thegraph_core::{DeploymentId, alloy::primitives::ChainId}; + +/// One captured lifecycle event: the routing metadata plus the typed payload. +#[derive(Clone, Debug, PartialEq)] +pub enum CapturedEvent { + RequestReceived { + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementRequestReceived, + }, + Proposed { + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementProposed, + }, + Accepted { + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementAccepted, + }, + RequestExpired { + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementRequestExpired, + }, + NIndexersUnavailable { + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementNIndexersUnavailable, + }, + Terminated { + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementTerminated, + }, +} + +/// A [`SubgraphIndexingAgreementEventsProducer`] that records produced events into +/// a shared buffer for assertions. Clone shares the same buffer. +#[derive(Clone, Default)] +pub struct CapturingEventsProducer { + events: Arc>>, +} + +impl CapturingEventsProducer { + pub fn new() -> Self { + Self::default() + } + + /// Snapshot of all events captured so far, in production order. + pub fn events(&self) -> Vec { + self.events.lock().expect("events lock poisoned").clone() + } + + fn push(&self, event: CapturedEvent) { + self.events + .lock() + .expect("events lock poisoned") + .push(event); + } +} + +#[async_trait] +impl SubgraphIndexingAgreementEventsProducer for CapturingEventsProducer { + fn produce_subgraph_indexing_agreement_request_received( + &self, + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementRequestReceived, + ) { + self.push(CapturedEvent::RequestReceived { + deployment, + chain_id, + event, + }); + } + + fn produce_subgraph_indexing_agreement_proposed( + &self, + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementProposed, + ) { + self.push(CapturedEvent::Proposed { + deployment, + chain_id, + event, + }); + } + + async fn emit_subgraph_indexing_agreement_accepted( + &self, + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementAccepted, + ) -> Result<(), EmitError> { + self.push(CapturedEvent::Accepted { + deployment, + chain_id, + event, + }); + Ok(()) + } + + async fn emit_subgraph_indexing_agreement_request_expired( + &self, + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementRequestExpired, + ) -> Result<(), EmitError> { + self.push(CapturedEvent::RequestExpired { + deployment, + chain_id, + event, + }); + Ok(()) + } + + fn produce_subgraph_indexing_agreement_n_indexers_unavailable( + &self, + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementNIndexersUnavailable, + ) { + self.push(CapturedEvent::NIndexersUnavailable { + deployment, + chain_id, + event, + }); + } + + async fn emit_subgraph_indexing_agreement_terminated( + &self, + deployment: DeploymentId, + chain_id: ChainId, + event: proto::SubgraphIndexingAgreementTerminated, + ) -> Result<(), EmitError> { + self.push(CapturedEvent::Terminated { + deployment, + chain_id, + event, + }); + Ok(()) + } +} diff --git a/bin/dipper-service/src/worker/context.rs b/bin/dipper-service/src/worker/context.rs index a7d8a44c..27be39a4 100644 --- a/bin/dipper-service/src/worker/context.rs +++ b/bin/dipper-service/src/worker/context.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; use dipper_core::state::FromState; +use dipper_producer::events::SubgraphIndexingAgreementEventsProducer; use graph_networks_registry::NetworksRegistry; use thegraph_core::alloy::primitives::ChainId; use tokio::sync::{Mutex, Notify}; @@ -125,6 +126,10 @@ pub struct Ctx { /// Number of concurrent worker loops to spawn (>=1). Defaults to 1. pub concurrency: usize, + + /// Subgraph Indexing Agreements Event emitter for sending indexing agreement events downstream + pub subgraph_indexing_agreements_events_emitter: + Arc, } /// The inner worker context. @@ -188,6 +193,10 @@ pub(super) struct InnerCtx { /// See `Ctx::dips_accepting_cache`. pub dips_accepting_cache: DipsAcceptingCache, + + /// See: `Ctx::subgraph_indexing_agreements_events_emitter` + pub subgraph_indexing_agreements_events_emitter: + Arc, } impl_from_state!(ReassessIndexingRequestCtx { @@ -209,6 +218,7 @@ impl_from_state!(ReassessIndexingRequestCtx { reassess_lock, unresponsive_breaker, dips_accepting_cache, + subgraph_indexing_agreements_events_emitter }); impl_from_state!(SendIndexingAgreementProposalCtx { diff --git a/bin/dipper-service/src/worker/handlers/cancel_rejected_agreement_on_chain.rs b/bin/dipper-service/src/worker/handlers/cancel_rejected_agreement_on_chain.rs index 5cebd408..a5cfc33d 100644 --- a/bin/dipper-service/src/worker/handlers/cancel_rejected_agreement_on_chain.rs +++ b/bin/dipper-service/src/worker/handlers/cancel_rejected_agreement_on_chain.rs @@ -35,7 +35,7 @@ pub struct Message { /// ensure the indexer doesn't receive payment for work we didn't want. pub async fn handle(ctx: Ctx, Message { agreement_id }: &Message) -> JobResult<()> where - R: AgreementRegistry, + R: AgreementRegistry + Sync, T: ChainClient, { // Look up the agreement @@ -74,44 +74,71 @@ where ); // Send the cancellation transaction (mode-aware dispatch). - match cancel_agreement_on_chain(&ctx.chain_client, &agreement, &ctx.agreement_conf).await { - Ok(Some(tx_hash)) => { - tracing::info!( - agreement_id = %agreement_id, - tx_hash = %tx_hash, - "Successfully submitted on-chain cancellation" - ); - mark_cancellation_complete(&ctx.registry, agreement_id).await; - Ok(()) - } - Ok(None) => { - tracing::info!( - agreement_id = %agreement_id, - "Rejected agreement already canceled on-chain; reconciling local state" - ); - mark_cancellation_complete(&ctx.registry, agreement_id).await; - Ok(()) - } - Err(err @ ChainClientError::MissingTermsVersionHash { .. }) => { - // Permanent: the hash never appears, so retrying can't help. Fail - // terminally and leave the live agreement for operator action. - tracing::error!( - agreement_id = %agreement_id, - error = %err, - "Cannot cancel rejected agreement: missing terms_version_hash" - ); - Err(JobError::Fatal(err.into())) - } - Err(err) => { + let on_chain_cancel_tx: Option = + match cancel_agreement_on_chain(&ctx.chain_client, &agreement, &ctx.agreement_conf).await { + Ok(Some(tx_hash)) => { + tracing::info!( + agreement_id = %agreement_id, + tx_hash = %tx_hash, + "Successfully submitted on-chain cancellation" + ); + Some(tx_hash.to_string()) + } + Ok(None) => { + tracing::info!( + agreement_id = %agreement_id, + "Rejected agreement already canceled on-chain; reconciling local state" + ); + None + } + Err(err @ ChainClientError::MissingTermsVersionHash { .. }) => { + // Permanent: the hash never appears, so retrying can't help. Fail + // terminally and leave the live agreement for operator action. + tracing::error!( + agreement_id = %agreement_id, + error = %err, + "Cannot cancel rejected agreement: missing terms_version_hash" + ); + return Err(JobError::Fatal(err.into())); + } + Err(err) => { + tracing::warn!( + agreement_id = %agreement_id, + error = %err, + "Failed to cancel agreement on-chain, will retry" + ); + // Retry with backoff - on-chain transactions can fail due to gas issues, nonce, etc. + return Err(JobError::Retryable(err.into(), Duration::from_secs(30))); + } + }; + + // When the row was actually flipped to terminal, record the cancel audit so + // the chain_listener's `terminated` sweep announces it durably. The accept + // was recorded when the rejected-then-accepted anomaly was first detected, so + // the row is sweep-eligible. If the mark failed, the row stays `Rejected` and + // the chain_listener observes the on-chain cancel and flips it itself, then + // the same sweep emits -- so nothing is lost either way. + if mark_cancellation_complete(&ctx.registry, agreement_id).await { + let manager = ctx.agreement_conf.recurring_agreement_manager().to_string(); + if let Err(err) = ctx + .registry + .record_cancel_audit( + agreement_id, + dipper_core::time::now_secs(), + &manager, + on_chain_cancel_tx.as_deref(), + ) + .await + { tracing::warn!( agreement_id = %agreement_id, error = %err, - "Failed to cancel agreement on-chain, will retry" + "failed to record cancel audit; terminated event may emit with fallback fields" ); - // Retry with backoff - on-chain transactions can fail due to gas issues, nonce, etc. - Err(JobError::Retryable(err.into(), Duration::from_secs(30))) } } + + Ok(()) } /// Flip the local row to CanceledByRequester after either a fresh on-chain @@ -119,26 +146,486 @@ where /// Failures here are logged but not fatal — the on-chain side is already in /// the right state, so the next reconciliation pass can re-attempt the DB /// update without risking a duplicate transaction. -async fn mark_cancellation_complete(registry: &R, agreement_id: &IndexingAgreementId) +/// +/// Returns `true` when the row was marked terminal. The caller emits the +/// `terminated` event only on `true`: if the mark failed the row stays +/// `Rejected` (non-terminal), so the chain_listener will observe the on-chain +/// cancel and emit `terminated` itself — emitting here too would duplicate. +async fn mark_cancellation_complete(registry: &R, agreement_id: &IndexingAgreementId) -> bool where - R: AgreementRegistry, + R: AgreementRegistry + Sync, { - if let Err(err) = registry + match registry .mark_indexing_agreement_as_canceled_by_requester(agreement_id) .await { - tracing::error!( - agreement_id = %agreement_id, - error = %err, - "Failed to update agreement status after on-chain cancellation" + Ok(()) => { + tracing::info!( + agreement_id = %agreement_id, + old_status = "REJECTED", + new_status = "CANCELED_BY_REQUESTER", + reason = "canceled_on_chain_after_rejection", + "agreement state transition" + ); + true + } + Err(err) => { + tracing::error!( + agreement_id = %agreement_id, + error = %err, + "Failed to update agreement status after on-chain cancellation" + ); + false + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use async_trait::async_trait; + use dipper_core::ids::IndexingRequestId; + use dipper_rpc::indexer::indexer_client::sol::RecurringCollectionAgreement; + use thegraph_core::{ + DeploymentId, IndexerId, + alloy::primitives::{Address, B256, U256}, + }; + use time::OffsetDateTime; + use url::Url; + + use super::*; + use crate::{ + chain_client::{ChainClient, ChainClientError}, + registry::{ + IndexingAgreement, IndexingAgreementStatus, IndexingAgreementTerms, + IndexingAgreementTermsMetadata, + }, + }; + + // ========================================================================= + // Mock implementations + // ========================================================================= + + /// Registry that returns a single configurable agreement and records the + /// terminal-cancel transition + cancel-audit calls the handler drives. + /// `Clone` shares the tracked state (Arc), so a test can clone one into the + /// `Ctx` and still assert on the original after `handle` consumes the ctx. + #[derive(Clone)] + struct MockRegistry { + agreement: Arc>>, + marked_canceled: Arc>>, + /// Ids passed to `record_cancel_audit` -- the signal the handler drives + /// the terminated event (the chain_listener sweep emits from this audit). + recorded_cancel_audit: Arc>>, + /// When true, `mark_indexing_agreement_as_canceled_by_requester` errors. + fail_mark: bool, + } + + impl MockRegistry { + fn new(agreement: IndexingAgreement) -> Self { + Self { + agreement: Arc::new(Mutex::new(Some(agreement))), + marked_canceled: Arc::new(Mutex::new(Vec::new())), + recorded_cancel_audit: Arc::new(Mutex::new(Vec::new())), + fail_mark: false, + } + } + + fn with_mark_failure(agreement: IndexingAgreement) -> Self { + Self { + fail_mark: true, + ..Self::new(agreement) + } + } + } + + #[async_trait] + impl AgreementRegistry for MockRegistry { + async fn get_indexing_agreement_by_id( + &self, + _id: &IndexingAgreementId, + ) -> crate::registry::Result> { + Ok(self.agreement.lock().unwrap().clone()) + } + + async fn get_indexing_agreements_by_deployment_id( + &self, + _deployment_id: &DeploymentId, + ) -> crate::registry::Result> { + Ok(vec![]) + } + + async fn get_indexing_agreements_by_indexer_id( + &self, + _indexer_id: &IndexerId, + ) -> crate::registry::Result> { + Ok(vec![]) + } + + async fn get_pending_agreement_indexers_by_deployment( + &self, + _indexer_ids: &[IndexerId], + ) -> crate::registry::Result>> + { + Ok(std::collections::HashMap::new()) + } + + async fn get_declined_indexers_by_deployment( + &self, + _default_lookback_days: i32, + _price_lookback_days: i32, + _transient_lookback_minutes: i32, + _uncertain_lookback_days: i32, + ) -> crate::registry::Result>> + { + Ok(std::collections::HashMap::new()) + } + + async fn get_indexing_agreements_by_indexing_request_id( + &self, + _request_id: &IndexingRequestId, + ) -> crate::registry::Result> { + Ok(vec![]) + } + + async fn get_active_indexing_agreements_by_indexing_request_id( + &self, + _request_id: &IndexingRequestId, + ) -> crate::registry::Result> { + Ok(vec![]) + } + + async fn count_accepted_agreements_by_deployment( + &self, + _deployment_id: &DeploymentId, + ) -> crate::registry::Result { + Ok(0) + } + + async fn record_cancel_audit( + &self, + agreement_id: &IndexingAgreementId, + _canceled_at: u64, + _canceled_by: &str, + _canceled_tx: Option<&str>, + ) -> crate::registry::Result<()> { + self.recorded_cancel_audit + .lock() + .unwrap() + .push(*agreement_id); + Ok(()) + } + + async fn register_new_indexing_agreement( + &self, + _params: crate::registry::NewAgreementParams, + ) -> crate::registry::Result { + Ok(IndexingAgreementId::from_bytes(rand::random())) + } + + async fn register_agreement_with_pending_cancellation( + &self, + _params: crate::registry::NewAgreementParams, + _old_agreement_id: IndexingAgreementId, + ) -> crate::registry::Result { + Ok(IndexingAgreementId::from_bytes(rand::random())) + } + + async fn get_unresponsive_indexers( + &self, + _lookback_days: i32, + _chain_id: thegraph_core::alloy::primitives::ChainId, + ) -> crate::registry::Result> { + unimplemented!() + } + + async fn mark_indexing_agreement_as_unresponsive( + &self, + _id: &IndexingAgreementId, + ) -> crate::registry::Result<()> { + unimplemented!() + } + + async fn count_created_agreements_by_indexer( + &self, + ) -> crate::registry::Result<(std::collections::HashMap, u64)> { + unimplemented!() + } + + async fn update_offer_tx_hash( + &self, + _id: &IndexingAgreementId, + _tx_hash: &[u8; 32], + ) -> crate::registry::Result<()> { + Ok(()) + } + + async fn mark_indexing_agreement_as_canceled_by_requester( + &self, + id: &IndexingAgreementId, + ) -> crate::registry::Result<()> { + if self.fail_mark { + return Err(crate::registry::Error::NoRecordsUpdated); + } + self.marked_canceled.lock().unwrap().push(*id); + Ok(()) + } + + async fn apply_reconciliation( + &self, + _id: &IndexingAgreementId, + _apply_accept: bool, + _cancel: Option, + ) -> crate::registry::Result { + Ok(crate::registry::ReconciliationOutcome { + did_accept: false, + did_cancel: false, + }) + } + + async fn get_expired_created_agreements( + &self, + _batch_size: i64, + _chain_timestamp: u64, + ) -> crate::registry::Result> { + Ok(vec![]) + } + + async fn mark_indexing_agreement_as_expired( + &self, + _id: &IndexingAgreementId, + ) -> crate::registry::Result<()> { + Ok(()) + } + + async fn mark_indexing_agreement_as_rejected( + &self, + _id: &IndexingAgreementId, + _rejection_reason: Option<&str>, + ) -> crate::registry::Result<()> { + Ok(()) + } + + async fn get_accepted_on_chain_agreements( + &self, + _batch_size: i64, + ) -> crate::registry::Result> { + Ok(vec![]) + } + + async fn get_agreements_pending_chain_cancel( + &self, + _batch_size: i64, + ) -> crate::registry::Result> { + Ok(vec![]) + } + + async fn update_agreement_sync_progress( + &self, + _id: &IndexingAgreementId, + _block_height: u64, + _progress_at: time::OffsetDateTime, + ) -> crate::registry::Result<()> { + Ok(()) + } + + async fn count_active_agreements_by_deployment( + &self, + ) -> crate::registry::Result> { + Ok(std::collections::HashMap::new()) + } + + async fn mark_indexing_agreement_as_abandoned( + &self, + _id: &IndexingAgreementId, + ) -> crate::registry::Result { + Err(crate::registry::Error::NoRecordsUpdated) + } + + async fn get_agreement_fee_rates( + &self, + ) -> crate::registry::Result> { + Ok(vec![]) + } + } + + /// Chain client whose manager cancel always mines (returns a tx hash) and + /// whose post-cancel liveness read reports the agreement is no longer active, + /// so the cancel is confirmed. + #[derive(Default)] + struct MockChainClient; + + #[async_trait] + impl ChainClient for MockChainClient { + async fn latest_block_timestamp(&self) -> Result { + Err(ChainClientError::RpcError(anyhow::anyhow!( + "latest_block_timestamp not mocked" + ))) + } + + async fn offer_via_manager( + &self, + _rca: &RecurringCollectionAgreement, + ) -> Result, ChainClientError> { + Ok(None) + } + + async fn cancel_via_manager( + &self, + _collector: Address, + _agreement_id: &[u8; 16], + _version_hash: B256, + _options: u16, + ) -> Result, ChainClientError> { + Ok(Some(B256::ZERO)) + } + + async fn reconcile_provider( + &self, + _collector: Address, + _provider: Address, + ) -> Result, ChainClientError> { + Ok(None) + } + + async fn agreement_still_active( + &self, + _agreement_id: &[u8; 16], + ) -> Result { + Ok(false) + } + } + + fn test_agreement_conf() -> Arc { + Arc::new(IndexingAgreementConfig { + data_service: Address::ZERO, + recurring_collector: Address::ZERO, + recurring_agreement_manager: Address::ZERO, + max_agreement_grt_per_30_days: 0.0, + max_seconds_per_collection: 0, + min_seconds_per_collection: 0, + duration_seconds: 0, + deadline_seconds: 0, + max_grt_per_30_days: std::collections::BTreeMap::new(), + max_grt_per_billion_entities_per_30_days: 0.0, + declined_indexer_lookback_days: 0, + price_rejection_lookback_days: 0, + transient_rejection_lookback_minutes: 0, + uncertain_rejection_lookback_days: 0, + unresponsive_indexer_lookback_days: 0, + mass_unresponsive_trip_fraction: 0.5, + mass_unresponsive_reset_fraction: 0.25, + dips_accepting_snapshot_max_age_hours: 48, + dips_accepting_cache_ttl_seconds: 300, + max_in_flight_offers_per_indexer: None, + max_in_flight_offers_total: None, + }) + } + + fn test_deployment_id() -> DeploymentId { + "QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9" + .parse() + .unwrap() + } + + fn make_agreement(status: IndexingAgreementStatus) -> IndexingAgreement { + IndexingAgreement { + id: IndexingAgreementId::from_bytes(rand::random()), + nonce_uuid: uuid::Uuid::now_v7(), + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + status, + indexing_request_id: IndexingRequestId::new(), + indexer: crate::registry::Indexer { + id: IndexerId::from(Address::ZERO), + url: Url::parse("https://indexer.example").unwrap(), + }, + terms: IndexingAgreementTerms { + payer: Address::ZERO, + service_provider: Address::ZERO, + data_service: Address::ZERO, + deadline: 0, + ends_at: 0, + max_initial_tokens: U256::ZERO, + max_ongoing_tokens_per_second: U256::ZERO, + min_seconds_per_collection: 0, + max_seconds_per_collection: 0, + conditions: 0, + metadata: IndexingAgreementTermsMetadata { + tokens_per_second: U256::ZERO, + tokens_per_entity_per_second: U256::ZERO, + subgraph_deployment_id: test_deployment_id(), + protocol_network: 1u64, + chain_id: 1u64, + proposed_at: 0, + }, + }, + last_block_height: None, + last_progress_at: None, + rejection_reason: None, + // 32-byte hash so the on-chain cancel path is exercised. + terms_version_hash: Some(vec![0u8; 32]), + } + } + + // ========================================================================= + // Tests + // ========================================================================= + + fn ctx_for(registry: MockRegistry) -> Ctx { + Ctx { + registry, + chain_client: MockChainClient, + agreement_conf: test_agreement_conf(), + } + } + + #[tokio::test] + async fn rejected_agreement_records_cancel_audit_once() { + // The handler no longer emits `terminated` directly: it records the cancel + // audit, and the chain_listener sweep announces it durably. Assert exactly + // one audit was recorded for the agreement. + let agreement = make_agreement(IndexingAgreementStatus::Rejected); + let agreement_id = agreement.id; + let registry = MockRegistry::new(agreement); + + let result = handle(ctx_for(registry.clone()), &Message { agreement_id }).await; + assert!(result.is_ok(), "handle should succeed: {result:?}"); + + let recorded = registry.recorded_cancel_audit.lock().unwrap().clone(); + assert_eq!(recorded, vec![agreement_id], "exactly one cancel audit"); + } + + #[tokio::test] + async fn failed_local_mark_records_no_cancel_audit() { + // On-chain cancel succeeds but the local DB mark fails, leaving the row + // non-terminal. The handler must NOT record cancel audit -- the + // chain_listener will observe the on-chain cancel, flip the row, and the + // sweep emits from there. + let agreement = make_agreement(IndexingAgreementStatus::Rejected); + let agreement_id = agreement.id; + let registry = MockRegistry::with_mark_failure(agreement); + + let result = handle(ctx_for(registry.clone()), &Message { agreement_id }).await; + assert!(result.is_ok(), "handle should still return Ok: {result:?}"); + assert!( + registry.recorded_cancel_audit.lock().unwrap().is_empty(), + "no cancel audit recorded when the local mark failed" ); - } else { - tracing::info!( - agreement_id = %agreement_id, - old_status = "REJECTED", - new_status = "CANCELED_BY_REQUESTER", - reason = "canceled_on_chain_after_rejection", - "agreement state transition" + } + + #[tokio::test] + async fn non_rejected_agreement_records_nothing() { + let agreement = make_agreement(IndexingAgreementStatus::Created); + let agreement_id = agreement.id; + let registry = MockRegistry::new(agreement); + + let result = handle(ctx_for(registry.clone()), &Message { agreement_id }).await; + assert!(result.is_ok(), "handle should return Ok: {result:?}"); + assert!( + registry.recorded_cancel_audit.lock().unwrap().is_empty(), + "no cancel audit recorded for a non-Rejected agreement" ); } } diff --git a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs index 142c3e3b..bcf8f25c 100644 --- a/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs +++ b/bin/dipper-service/src/worker/handlers/reassess_indexing_request.rs @@ -6,6 +6,7 @@ use std::{ use dipper_core::{ids::IndexingRequestId, time::now_secs}; use dipper_iisa::{CandidateSelection, SelectedIndexer, SelectionError}; +use dipper_producer::{events::SubgraphIndexingAgreementEventsProducer, proto}; use graph_networks_registry::NetworksRegistry; use jsonrpsee::core::Serialize; use serde::Deserialize; @@ -75,6 +76,8 @@ pub struct Ctx { pub unresponsive_breaker: Arc, /// Cache of IISA's DIPs-accepting set (the breaker's denominator). pub dips_accepting_cache: DipsAcceptingCache, + pub subgraph_indexing_agreements_events_emitter: + Arc, } /// Reassess an indexing request against the current IISA target state. @@ -108,7 +111,8 @@ where + AgreementRegistry + IndexerDenylistRegistry + PendingCancellationRegistry - + ChainListenerStateRegistry, + + ChainListenerStateRegistry + + Sync, W: WorkerQueue, I: CandidateSelection, T: ChainClient, @@ -281,6 +285,40 @@ where "reassessment diff computed" ); + // Coverage-shortfall signal (transition-based). IISA returning fewer + // candidates than requested means the request cannot be fully covered. We + // fire `n_indexers_unavailable` ONCE, on the transition INTO shortfall, and + // clear the latch on recovery -- so it covers the zero-available and + // coverage-drop cases (which produce an empty diff and would otherwise be + // silent) without re-emitting on every recurring reassessment of a + // persistently short request. The atomic latch update makes this retry-safe: + // a re-run after a later failure sees the latch already set and does not + // re-emit. + let in_shortfall = target_selected.len() < *num_candidates; + let shortfall_changed = ctx + .registry + .set_indexing_request_shortfall_active(indexing_request_id, in_shortfall) + .await + .unwrap_or_else(|err| { + tracing::warn!( + indexing_request_id=%indexing_request_id, + error=%err, + "failed to update shortfall latch; skipping shortfall signal this cycle" + ); + false + }); + if in_shortfall && shortfall_changed { + ctx.subgraph_indexing_agreements_events_emitter + .produce_subgraph_indexing_agreement_n_indexers_unavailable( + *deployment_id, + ctx.signer.chain_id(), + proto::SubgraphIndexingAgreementNIndexersUnavailable { + agreements_requested: *num_candidates as i32, + candidates_returned: target_selected.len() as i32, + }, + ); + } + if to_cancel.is_empty() && to_add.is_empty() { tracing::debug!( indexing_request_id=%indexing_request_id, @@ -361,6 +399,9 @@ where let mut old_iter = old_to_cancel.into_iter(); let mut successful_new_ids: Vec = vec![]; + // 0x addresses of the indexers we successfully sent proposals to this cycle, + // for the `proposed` lifecycle event emitted after the loop. + let mut proposed_candidates: Vec = vec![]; let mut add_failures = 0u32; let mut _pending_recorded = 0u32; for indexer_id in &capped_to_add { @@ -405,6 +446,8 @@ where subgraph_deployment_id: *deployment_id, protocol_network: ctx.signer.chain_id(), chain_id: *deployment_chain_id, + // Same clock as `deadline` below, so the expiry event's two timestamps agree. + proposed_at: now, }; // The RCA's contract caps don't need to be per-chain. At large @@ -562,9 +605,26 @@ where "proposal queued" ); successful_new_ids.push(agreement_id); + proposed_candidates.push(indexer_id.to_string()); } } + // Offers were sent to the IISA candidates (but none accepted on-chain yet): + // emit the `proposed` lifecycle event with the candidates proposed this cycle + // and the acceptance deadline. Recurring reassessments re-emit when new + // additions go out. + if !proposed_candidates.is_empty() { + ctx.subgraph_indexing_agreements_events_emitter + .produce_subgraph_indexing_agreement_proposed( + *deployment_id, + ctx.signer.chain_id(), + proto::SubgraphIndexingAgreementProposed { + candidates: proposed_candidates, + request_expires_at: now.saturating_add(ctx.agreement_conf.deadline_seconds()), + }, + ); + } + // Cancel old agreements that have no replacement to pair with. // These indexers are leaving the target group with nothing taking // their place. Fire the on-chain cancel first; only mark the local row @@ -581,6 +641,7 @@ where crate::registry::IndexingAgreementStatus::AcceptedOnChain ); + let mut on_chain_cancel_tx: Option = None; if needs_on_chain_cancel { match crate::cancel_dispatch::cancel_agreement_on_chain( &ctx.chain_client, @@ -596,6 +657,7 @@ where %tx_hash, "Submitted on-chain cancellation for unpaired old agreement" ); + on_chain_cancel_tx = Some(tx_hash.to_string()); } Ok(None) => { tracing::info!( @@ -639,6 +701,31 @@ where "agreement state transition" ); + // Record the cancel audit for the accepted-on-chain agreements dipper just + // cancelled, so the chain_listener's `terminated` sweep announces them + // durably. Never-accepted agreements (`!needs_on_chain_cancel`) were never + // live on-chain: they are not sweep-eligible (`accepted_at IS NULL`) and + // correctly emit nothing. + if needs_on_chain_cancel { + let manager = ctx.agreement_conf.recurring_agreement_manager().to_string(); + if let Err(err) = ctx + .registry + .record_cancel_audit( + &old_agreement.id, + now_secs(), + &manager, + on_chain_cancel_tx.as_deref(), + ) + .await + { + tracing::warn!( + agreement_id = %old_agreement.id, + error = %err, + "failed to record cancel audit; terminated event may emit with fallback fields" + ); + } + } + directly_cancelled += 1; } @@ -784,6 +871,1025 @@ where } } +#[cfg(test)] +mod lifecycle_event_tests { + //! Validates the three lifecycle-event emits of [`handle`]: + //! `n_indexers_unavailable`, `proposed`, and `terminated`. + //! + //! Each test drives the real handler against in-memory mocks of every + //! generic bound (registry, queue, IISA, chain client) plus a + //! [`CapturingEventsProducer`] so it can assert exactly which events were + //! emitted (and with which payloads), and that no others fire. + + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; + + use async_trait::async_trait; + use dipper_core::ids::{IndexingAgreementId, IndexingRequestId}; + use dipper_iisa::{CandidateSelection, SelectedIndexer, SelectionContext, SelectionError}; + use dipper_rpc::indexer::indexer_client::sol::RecurringCollectionAgreement; + use thegraph_core::{ + DeploymentId, IndexerId, + alloy::{ + primitives::{Address, B256, ChainId, U256}, + sol_types::Eip712Domain, + }, + }; + use time::OffsetDateTime; + use url::Url; + + use super::{Ctx, Message, handle}; + use crate::{ + chain_client::{ChainClient, ChainClientError}, + config::IndexingAgreementConfig, + network::{ + provider::NetworkProviderService, + service::{ + chain_listener::{ChainListenerState, ChainListenerStateRegistry}, + indexer_urls, + }, + }, + registry::{ + AgreementFeeRate, AgreementRegistry, CancelKind, Indexer, IndexerDenylistRegistry, + IndexingAgreement, IndexingAgreementStatus, IndexingAgreementTerms, + IndexingAgreementTermsMetadata, IndexingRequest, IndexingRequestRegistry, + NewAgreementParams, PendingCancellation, PendingCancellationRegistry, + ReconciliationItem, ReconciliationOutcome, Result as RegistryResult, SetTargetOutcome, + }, + signing::eip712::Eip712Signer, + test_support::{CapturedEvent, CapturingEventsProducer}, + }; + + const TEST_PROTOCOL_CHAIN_ID: ChainId = 42161; + const TEST_DEPLOYMENT_CHAIN_ID: ChainId = 1; + + fn deployment() -> DeploymentId { + "QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9" + .parse() + .unwrap() + } + + fn indexer_id(byte: u8) -> IndexerId { + IndexerId::from(Address::repeat_byte(byte)) + } + + fn indexer_url() -> Url { + Url::parse("https://indexer.example").unwrap() + } + + // ---- Mock: IISA candidate selection ------------------------------------- + + /// Returns a fixed list of [`SelectedIndexer`] regardless of arguments. + struct MockIisa { + selected: Vec, + } + + #[async_trait] + impl CandidateSelection for MockIisa { + async fn select_indexers( + &self, + _deployment_id: DeploymentId, + _num_candidates: usize, + _context: &SelectionContext, + ) -> std::result::Result, SelectionError> { + Ok(self.selected.clone()) + } + + async fn dips_accepting_indexers( + &self, + _chain: &str, + _max_grt_per_30_days: Option, + ) -> std::result::Result { + Ok(dipper_iisa::DipsAcceptingSnapshot { + computed_at: None, + indexers: Vec::new(), + }) + } + } + + // ---- Mock: worker queue -------------------------------------------------- + + /// Records every `send_indexing_agreement_proposal` call's indexer URL. + /// Clone shares the buffer so a caller can inspect proposals after `handle`. + #[derive(Default, Clone)] + struct MockQueue { + proposals: Arc>>, + } + + #[async_trait] + impl crate::worker::service::WorkerQueue for MockQueue { + async fn send_indexing_agreement_proposal( + &self, + candidate_url: Url, + _agreement_id: IndexingAgreementId, + _indexing_request_id: IndexingRequestId, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + self.proposals.lock().unwrap().push(candidate_url); + Ok(crate::worker::queue::JobId::default()) + } + + async fn reassess_indexing_request( + &self, + _indexing_request_id: IndexingRequestId, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _num_candidates: usize, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + unimplemented!("not exercised by reassess handler") + } + + async fn cancel_rejected_agreement_on_chain( + &self, + _agreement_id: IndexingAgreementId, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + unimplemented!("not exercised by reassess handler") + } + + async fn submit_offer( + &self, + _agreement_id: IndexingAgreementId, + _indexing_request_id: IndexingRequestId, + _indexer_url: Url, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _priority: crate::worker::queue::JobPriority, + ) -> anyhow::Result { + unimplemented!("not exercised by reassess handler") + } + } + + // ---- Mock: chain client -------------------------------------------------- + + /// Always reports a successful cancel that the post-cancel read confirms. + #[derive(Default)] + struct MockChainClient; + + #[async_trait] + impl ChainClient for MockChainClient { + async fn latest_block_timestamp(&self) -> std::result::Result { + Err(ChainClientError::RpcError(anyhow::anyhow!( + "latest_block_timestamp not mocked" + ))) + } + + async fn offer_via_manager( + &self, + _rca: &RecurringCollectionAgreement, + ) -> std::result::Result, ChainClientError> { + Ok(None) + } + + async fn cancel_via_manager( + &self, + _collector: Address, + _agreement_id: &[u8; 16], + _version_hash: B256, + _options: u16, + ) -> std::result::Result, ChainClientError> { + Ok(Some(B256::repeat_byte(0xcd))) + } + + async fn reconcile_provider( + &self, + _collector: Address, + _provider: Address, + ) -> std::result::Result, ChainClientError> { + Ok(None) + } + + async fn agreement_still_active( + &self, + _agreement_id: &[u8; 16], + ) -> std::result::Result { + // Cancel confirmed: agreement is no longer active on-chain. + Ok(false) + } + } + + // ---- Mock: registry (all five traits) ----------------------------------- + + /// In-memory registry seeded with the active agreements for the request. + /// Only the methods the exercised reassess path touches return data; the + /// rest `unimplemented!()`. + #[derive(Default)] + struct MockRegistry { + active_agreements: Vec, + accepted_count: i64, + /// When true, `get_chain_listener_state` errors, driving `resolve_deadline_clock` + /// onto its retry path -- used to assert the shortfall event isn't emitted first. + chain_state_lookup_fails: bool, + /// Coverage-shortfall latch, mirroring the DB column (starts `false`). + /// `set_indexing_request_shortfall_active` flips it and reports whether it + /// changed, so tests can exercise the transition-based emit. + shortfall_active: std::sync::Mutex, + } + + #[async_trait] + impl IndexingRequestRegistry for MockRegistry { + async fn set_indexing_target_candidates( + &self, + _requested_by: Address, + _deployment_id: DeploymentId, + _deployment_chain_id: ChainId, + _num_candidates: usize, + ) -> RegistryResult { + unimplemented!() + } + async fn get_all_indexing_requests(&self) -> RegistryResult> { + unimplemented!() + } + async fn get_indexing_request_by_id( + &self, + _id: &IndexingRequestId, + ) -> RegistryResult> { + unimplemented!() + } + async fn get_indexing_requests_by_deployment_id( + &self, + _deployment_id: &DeploymentId, + ) -> RegistryResult> { + unimplemented!() + } + async fn get_open_indexing_requests_for_reassessment( + &self, + _min_age_seconds: i64, + _batch_size: i64, + ) -> RegistryResult> { + unimplemented!() + } + async fn set_indexing_request_shortfall_active( + &self, + _id: &IndexingRequestId, + active: bool, + ) -> RegistryResult { + let mut latch = self.shortfall_active.lock().unwrap(); + let changed = *latch != active; + *latch = active; + Ok(changed) + } + } + + #[async_trait] + impl AgreementRegistry for MockRegistry { + async fn get_indexing_agreement_by_id( + &self, + _id: &IndexingAgreementId, + ) -> RegistryResult> { + unimplemented!() + } + // gather_selection_context: all active agreements for the deployment. + async fn get_indexing_agreements_by_deployment_id( + &self, + _deployment_id: &DeploymentId, + ) -> RegistryResult> { + Ok(self.active_agreements.clone()) + } + async fn get_indexing_agreements_by_indexer_id( + &self, + _indexer_id: &IndexerId, + ) -> RegistryResult> { + unimplemented!() + } + // gather_selection_context: no pending agreements. + async fn get_pending_agreement_indexers_by_deployment( + &self, + _indexer_ids: &[IndexerId], + ) -> RegistryResult>> { + Ok(HashMap::new()) + } + // gather_selection_context: no declined indexers. + async fn get_declined_indexers_by_deployment( + &self, + _default_lookback_days: i32, + _price_lookback_days: i32, + _transient_lookback_minutes: i32, + _uncertain_lookback_days: i32, + ) -> RegistryResult>> { + Ok(HashMap::new()) + } + async fn get_indexing_agreements_by_indexing_request_id( + &self, + _request_id: &IndexingRequestId, + ) -> RegistryResult> { + unimplemented!() + } + // The handler's current-state baseline for the diff. + async fn get_active_indexing_agreements_by_indexing_request_id( + &self, + _request_id: &IndexingRequestId, + ) -> RegistryResult> { + Ok(self.active_agreements.clone()) + } + // Populates the `terminated` event's remaining count. + async fn count_accepted_agreements_by_deployment( + &self, + _deployment_id: &DeploymentId, + ) -> RegistryResult { + Ok(self.accepted_count) + } + async fn register_new_indexing_agreement( + &self, + params: NewAgreementParams, + ) -> RegistryResult { + Ok(params.agreement_id) + } + async fn register_agreement_with_pending_cancellation( + &self, + params: NewAgreementParams, + _old_agreement_id: IndexingAgreementId, + ) -> RegistryResult { + Ok(params.agreement_id) + } + async fn get_unresponsive_indexers( + &self, + _lookback_days: i32, + _chain_id: ChainId, + ) -> RegistryResult> { + Ok(Vec::new()) + } + async fn mark_indexing_agreement_as_unresponsive( + &self, + _id: &IndexingAgreementId, + ) -> RegistryResult<()> { + unimplemented!() + } + async fn count_created_agreements_by_indexer( + &self, + ) -> RegistryResult<(std::collections::HashMap, u64)> { + Ok((std::collections::HashMap::new(), 0)) + } + async fn update_offer_tx_hash( + &self, + _id: &IndexingAgreementId, + _tx_hash: &[u8; 32], + ) -> RegistryResult<()> { + unimplemented!() + } + // Cancel path: pre-mark the local row terminal. + async fn mark_indexing_agreement_as_canceled_by_requester( + &self, + _id: &IndexingAgreementId, + ) -> RegistryResult<()> { + Ok(()) + } + async fn apply_reconciliation( + &self, + _id: &IndexingAgreementId, + _apply_accept: bool, + _cancel: Option, + ) -> RegistryResult { + unimplemented!() + } + async fn apply_reconciliation_batch( + &self, + _items: &[ReconciliationItem], + ) -> RegistryResult> { + unimplemented!() + } + async fn get_expired_created_agreements( + &self, + _batch_size: i64, + _chain_timestamp: u64, + ) -> RegistryResult> { + unimplemented!() + } + async fn mark_indexing_agreement_as_expired( + &self, + _id: &IndexingAgreementId, + ) -> RegistryResult<()> { + unimplemented!() + } + async fn mark_indexing_agreement_as_rejected( + &self, + _id: &IndexingAgreementId, + _rejection_reason: Option<&str>, + ) -> RegistryResult<()> { + unimplemented!() + } + async fn get_accepted_on_chain_agreements( + &self, + _batch_size: i64, + ) -> RegistryResult> { + unimplemented!() + } + async fn get_agreements_pending_chain_cancel( + &self, + _batch_size: i64, + ) -> RegistryResult> { + unimplemented!() + } + async fn update_agreement_sync_progress( + &self, + _id: &IndexingAgreementId, + _block_height: u64, + _progress_at: OffsetDateTime, + ) -> RegistryResult<()> { + unimplemented!() + } + async fn count_active_agreements_by_deployment( + &self, + ) -> RegistryResult> { + unimplemented!() + } + async fn mark_indexing_agreement_as_abandoned( + &self, + _id: &IndexingAgreementId, + ) -> RegistryResult { + unimplemented!() + } + // gather_selection_context: optimistic DIPs fees (none). + async fn get_agreement_fee_rates(&self) -> RegistryResult> { + Ok(Vec::new()) + } + } + + #[async_trait] + impl IndexerDenylistRegistry for MockRegistry { + // gather_selection_context: empty denylist. + async fn get_indexer_denylist(&self) -> RegistryResult> { + Ok(Vec::new()) + } + } + + #[async_trait] + impl PendingCancellationRegistry for MockRegistry { + async fn get_pending_cancellations_by_new_agreement( + &self, + _new_agreement_id: IndexingAgreementId, + ) -> RegistryResult> { + unimplemented!() + } + async fn delete_pending_cancellations_by_new_agreement( + &self, + _new_agreement_id: IndexingAgreementId, + ) -> RegistryResult<()> { + unimplemented!() + } + async fn delete_pending_cancellation( + &self, + _new_agreement_id: IndexingAgreementId, + _old_agreement_id: IndexingAgreementId, + ) -> RegistryResult<()> { + unimplemented!() + } + async fn list_executable_pending_cancellations( + &self, + _limit: i64, + ) -> RegistryResult> { + unimplemented!() + } + } + + #[async_trait] + impl ChainListenerStateRegistry for MockRegistry { + async fn get_chain_listener_state( + &self, + _chain_id: u64, + ) -> RegistryResult> { + if self.chain_state_lookup_fails { + return Err(crate::registry::Error::NoRecordsUpdated); + } + Ok(None) + } + async fn update_chain_listener_state( + &self, + _chain_id: u64, + _cursor: &crate::network::service::chain_events::Cursor, + _last_processed_block_timestamp: Option, + ) -> RegistryResult<()> { + unimplemented!() + } + } + + // ---- Test fixtures ------------------------------------------------------- + + fn test_signer() -> Eip712Signer { + Eip712Signer::new( + TEST_PROTOCOL_CHAIN_ID, + Eip712Domain::new( + Some("RecurringCollector".into()), + Some("1".into()), + Some(U256::from(TEST_PROTOCOL_CHAIN_ID)), + Some(Address::repeat_byte(0x42)), + None, + ), + ) + } + + fn test_agreement_conf() -> IndexingAgreementConfig { + IndexingAgreementConfig { + data_service: Address::repeat_byte(0x03), + recurring_collector: Address::repeat_byte(0x04), + recurring_agreement_manager: Address::repeat_byte(0x05), + max_agreement_grt_per_30_days: 1000.0, + max_seconds_per_collection: 3600, + min_seconds_per_collection: 60, + duration_seconds: 86400, + deadline_seconds: 3600, + max_grt_per_30_days: std::collections::BTreeMap::new(), + max_grt_per_billion_entities_per_30_days: 0.0, + declined_indexer_lookback_days: 30, + price_rejection_lookback_days: 1, + transient_rejection_lookback_minutes: 30, + uncertain_rejection_lookback_days: 1, + unresponsive_indexer_lookback_days: 0, + mass_unresponsive_trip_fraction: 0.5, + mass_unresponsive_reset_fraction: 0.25, + dips_accepting_snapshot_max_age_hours: 48, + dips_accepting_cache_ttl_seconds: 300, + max_in_flight_offers_per_indexer: None, + max_in_flight_offers_total: None, + } + } + + fn empty_networks_registry() -> graph_networks_registry::NetworksRegistry { + graph_networks_registry::NetworksRegistry::from_json( + r#"{ + "$schema": "https://example/schema.json", + "description": "test", + "networks": [], + "title": "test", + "updatedAt": "2024-01-01T00:00:00Z", + "version": "0.0.0" + }"#, + ) + .expect("empty registry") + } + + /// Build a `Ctx` wired with the given mocks plus a seeded indexer-URL snapshot. + /// Returns the assembled `Ctx`. `events` is shared (clone) so the caller can + /// assert on it after `handle` runs. + #[allow(clippy::type_complexity)] + fn build_ctx( + registry: MockRegistry, + iisa: MockIisa, + queue: MockQueue, + chain_client: MockChainClient, + events: CapturingEventsProducer, + snapshot: indexer_urls::Snapshot, + ) -> Ctx { + let network = NetworkProviderService::new(indexer_urls::Handle::for_test(snapshot)); + Ctx { + signer: Arc::new(test_signer()), + agreement_conf: Arc::new(test_agreement_conf()), + rca_domain: Arc::new(std::sync::RwLock::new(Eip712Domain::new( + Some("RecurringCollector".into()), + Some("1".into()), + Some(U256::from(TEST_PROTOCOL_CHAIN_ID)), + Some(Address::repeat_byte(0x42)), + None, + ))), + chain_price: Arc::new(std::collections::BTreeMap::new()), + registry, + network, + queue, + iisa, + chain_client, + networks_registry: Arc::new(empty_networks_registry()), + // The breaker resolves the deployment chain's *name* before IISA + // selection; without a mapping the handler fails fatally up front. + additional_networks: Arc::new(std::collections::BTreeMap::from([( + TEST_DEPLOYMENT_CHAIN_ID, + "test-chain".to_string(), + )])), + entity_count_cache: crate::network::service::entity_count_cache::new_cache(), + chain_listener_notify: Arc::new(tokio::sync::Notify::new()), + bypass_chain_clock_defenses: false, + chain_listener_chain_id: None, + reassess_lock: Arc::new(tokio::sync::Mutex::new(())), + unresponsive_breaker: Arc::new(crate::worker::UnresponsiveBreaker::new()), + dips_accepting_cache: crate::worker::DipsAcceptingCache::new( + std::time::Duration::from_secs(300), + ), + subgraph_indexing_agreements_events_emitter: Arc::new(events), + } + } + + fn test_message(num_candidates: usize) -> Message { + Message { + indexing_request_id: IndexingRequestId::new(), + deployment_id: deployment(), + deployment_chain_id: TEST_DEPLOYMENT_CHAIN_ID, + num_candidates, + } + } + + /// Build an active agreement for `indexer` in the given status. + fn agreement(indexer: IndexerId, status: IndexingAgreementStatus) -> IndexingAgreement { + IndexingAgreement { + id: IndexingAgreementId::from_bytes(rand::random()), + nonce_uuid: uuid::Uuid::now_v7(), + created_at: OffsetDateTime::now_utc(), + updated_at: OffsetDateTime::now_utc(), + status, + indexing_request_id: IndexingRequestId::new(), + indexer: Indexer { + id: indexer, + url: indexer_url(), + }, + terms: IndexingAgreementTerms { + payer: Address::ZERO, + service_provider: Address::ZERO, + data_service: Address::ZERO, + deadline: 0, + ends_at: 0, + max_initial_tokens: U256::ZERO, + max_ongoing_tokens_per_second: U256::ZERO, + min_seconds_per_collection: 0, + max_seconds_per_collection: 0, + conditions: 0, + metadata: IndexingAgreementTermsMetadata { + tokens_per_second: U256::ZERO, + tokens_per_entity_per_second: U256::ZERO, + subgraph_deployment_id: deployment(), + protocol_network: TEST_PROTOCOL_CHAIN_ID, + chain_id: TEST_DEPLOYMENT_CHAIN_ID, + proposed_at: 0, + }, + }, + last_block_height: None, + last_progress_at: None, + rejection_reason: None, + // 32-byte hash so the on-chain cancel path can build a version hash. + terms_version_hash: Some(vec![7u8; 32]), + } + } + + /// A `SelectedIndexer` with IISA-provided pricing so `resolve_pricing` succeeds. + fn selected(id: IndexerId) -> SelectedIndexer { + SelectedIndexer { + id, + min_grt_per_30_days: Some(100.0), + min_grt_per_billion_entities_per_30_days: Some(1.0), + } + } + + // ---- Scenario 1: n_indexers_unavailable --------------------------------- + + #[tokio::test] + async fn emits_n_indexers_unavailable_when_iisa_returns_fewer() { + // Request of 3, IISA returns 1 resolvable candidate, no current + // agreements: the diff has a real addition (to_add = {new}) AND the + // selection falls short (1 < 3), so the handler emits exactly one + // NIndexersUnavailable (alongside the Proposed for the queued add). + let new_idx = indexer_id(0x21); + let mut snapshot = indexer_urls::Snapshot::new(); + snapshot.insert(new_idx, indexer_url()); + + let events = CapturingEventsProducer::new(); + let ctx = build_ctx( + MockRegistry::default(), + MockIisa { + selected: vec![selected(new_idx)], + }, + MockQueue::default(), + MockChainClient, + events.clone(), + snapshot, + ); + + handle(ctx, &test_message(3)).await.expect("handler ok"); + + let captured = events.events(); + let unavailable: Vec<_> = captured + .iter() + .filter_map(|e| match e { + CapturedEvent::NIndexersUnavailable { + deployment: d, + chain_id, + event, + } => Some((d, chain_id, event)), + _ => None, + }) + .collect(); + assert_eq!( + unavailable.len(), + 1, + "exactly one NIndexersUnavailable: {captured:?}" + ); + let (d, chain_id, event) = unavailable[0]; + assert_eq!(*d, deployment()); + assert_eq!(*chain_id, TEST_PROTOCOL_CHAIN_ID); + assert_eq!(event.agreements_requested, 3); + assert_eq!(event.candidates_returned, 1); + } + + // ---- Scenario 1b: shortfall transition semantics ------------------------ + + #[tokio::test] + async fn shortfall_fires_up_front_even_if_deadline_clock_read_fails() { + // The shortfall signal is evaluated up front (before the diff is applied + // and before `resolve_deadline_clock`), so entering shortfall fires the + // event even if a later step -- here the bypass-path deadline-clock read -- + // fails the job. The latch makes the subsequent retry idempotent. + let new_idx = indexer_id(0x21); + let mut snapshot = indexer_urls::Snapshot::new(); + snapshot.insert(new_idx, indexer_url()); + + let events = CapturingEventsProducer::new(); + let mut ctx = build_ctx( + MockRegistry { + chain_state_lookup_fails: true, + ..Default::default() + }, + MockIisa { + selected: vec![selected(new_idx)], + }, + MockQueue::default(), + MockChainClient, + events.clone(), + snapshot, + ); + ctx.bypass_chain_clock_defenses = true; + ctx.chain_listener_chain_id = Some(1337); + + // Request 3, IISA returns 1: enters shortfall. The clock read then fails. + let result = handle(ctx, &test_message(3)).await; + + assert!( + result.is_err(), + "a failed deadline-clock read must fail the job" + ); + let shortfall_count = events + .events() + .iter() + .filter(|e| matches!(e, CapturedEvent::NIndexersUnavailable { .. })) + .count(); + assert_eq!( + shortfall_count, + 1, + "shortfall fires up front, before the fallible clock read: {:?}", + events.events() + ); + } + + #[tokio::test] + async fn no_re_emit_when_already_in_shortfall() { + // A request already in shortfall (latch set) that stays short MUST NOT + // re-emit on a recurring reassessment -- the signal fires only on the + // transition INTO shortfall. + let idx = indexer_id(0x11); + let events = CapturingEventsProducer::new(); + let registry = MockRegistry { + active_agreements: vec![agreement(idx, IndexingAgreementStatus::AcceptedOnChain)], + accepted_count: 1, + chain_state_lookup_fails: false, + // Already in shortfall. + shortfall_active: std::sync::Mutex::new(true), + }; + let ctx = build_ctx( + registry, + MockIisa { + selected: vec![selected(idx)], + }, + MockQueue::default(), + MockChainClient, + events.clone(), + indexer_urls::Snapshot::new(), + ); + + // Request 3, IISA returns 1: still short, synced (empty diff). The latch is + // already set, so no re-emit. + handle(ctx, &test_message(3)).await.expect("handler ok"); + + assert!( + events.events().is_empty(), + "must not re-emit while already in shortfall: {:?}", + events.events() + ); + } + + // ---- Scenario 2: negative (no shortfall, empty diff) -------------------- + + #[tokio::test] + async fn no_n_indexers_unavailable_when_target_matches_current() { + // IISA returns >= num_candidates AND the target equals the current + // agreements, so the diff is empty: no event of any kind. + let idx = indexer_id(0x11); + let events = CapturingEventsProducer::new(); + let registry = MockRegistry { + active_agreements: vec![agreement(idx, IndexingAgreementStatus::AcceptedOnChain)], + accepted_count: 1, + chain_state_lookup_fails: false, + shortfall_active: std::sync::Mutex::new(false), + }; + let ctx = build_ctx( + registry, + MockIisa { + selected: vec![selected(idx)], + }, + MockQueue::default(), + MockChainClient, + events.clone(), + indexer_urls::Snapshot::new(), + ); + + handle(ctx, &test_message(1)).await.expect("handler ok"); + + assert!( + events.events().is_empty(), + "no events when diff is empty and no shortfall: {:?}", + events.events() + ); + } + + #[tokio::test] + async fn emits_shortfall_when_zero_indexers_available() { + // Item 3: a brand-new request for which IISA returns ZERO candidates -- the + // worst-case shortage. The diff is empty (nothing to add or cancel), which + // the old `!to_add` gate left silent. The transition model fires on + // entering shortfall regardless of the empty diff. + let events = CapturingEventsProducer::new(); + let ctx = build_ctx( + MockRegistry::default(), // no current agreements, latch false + MockIisa { selected: vec![] }, // zero available + MockQueue::default(), + MockChainClient, + events.clone(), + indexer_urls::Snapshot::new(), + ); + + handle(ctx, &test_message(3)).await.expect("handler ok"); + + let captured = events.events(); + let unavailable: Vec<_> = captured + .iter() + .filter_map(|e| match e { + CapturedEvent::NIndexersUnavailable { event, .. } => Some(event), + _ => None, + }) + .collect(); + assert_eq!( + unavailable.len(), + 1, + "zero-available must emit exactly one shortfall: {captured:?}" + ); + assert_eq!(unavailable[0].agreements_requested, 3); + assert_eq!(unavailable[0].candidates_returned, 0); + } + + #[tokio::test] + async fn emits_shortfall_when_coverage_drops_below_target() { + // Item 9: an indexer leaves the target group and IISA can't replace it, so + // coverage falls below the requested count. The diff is cancel-only + // (to_add empty), which the old gate left silent for a proposed-only + // leaver. The transition model fires the shortfall. + let leaving = indexer_id(0x11); + let events = CapturingEventsProducer::new(); + let registry = MockRegistry { + // A proposed-but-not-yet-accepted indexer that is leaving the group. + active_agreements: vec![agreement(leaving, IndexingAgreementStatus::Created)], + accepted_count: 0, + chain_state_lookup_fails: false, + shortfall_active: std::sync::Mutex::new(false), + }; + let ctx = build_ctx( + registry, + MockIisa { selected: vec![] }, // IISA returns nothing -> coverage drops + MockQueue::default(), + MockChainClient, + events.clone(), + indexer_urls::Snapshot::new(), + ); + + handle(ctx, &test_message(3)).await.expect("handler ok"); + + let unavailable = events + .events() + .into_iter() + .filter(|e| matches!(e, CapturedEvent::NIndexersUnavailable { .. })) + .count(); + assert_eq!( + unavailable, 1, + "coverage drop below target must emit exactly one shortfall" + ); + } + + // ---- Scenario 3: proposed on add; terminated is NOT emitted here -------- + + #[tokio::test] + async fn emits_proposed_on_add_and_does_not_emit_terminated_for_accepted_cancel() { + // Target = {new}, current = {old_paired, old_unpaired} (both + // AcceptedOnChain). With one add and two cancels, the add loop pairs the + // new agreement with the FIRST old agreement (atomic replacement), and the + // second, unpaired old agreement reaches the cancel loop. It was + // AcceptedOnChain, so reassess records its cancel audit -- but does NOT + // emit `terminated` (the chain_listener sweep is the sole emitter now). + // Net: exactly one `proposed`, zero `terminated`. + let new_idx = indexer_id(0x22); + let old_paired = indexer_id(0x33); + let old_unpaired = indexer_id(0x34); + + // Topology must resolve the new indexer or the add is skipped. + let mut snapshot = indexer_urls::Snapshot::new(); + snapshot.insert(new_idx, indexer_url()); + + let events = CapturingEventsProducer::new(); + let queue = MockQueue::default(); + let registry = MockRegistry { + // Vec order drives the pairing: first is paired, second is unpaired. + active_agreements: vec![ + agreement(old_paired, IndexingAgreementStatus::AcceptedOnChain), + agreement(old_unpaired, IndexingAgreementStatus::AcceptedOnChain), + ], + accepted_count: 0, + chain_state_lookup_fails: false, + shortfall_active: std::sync::Mutex::new(false), + }; + let ctx = build_ctx( + registry, + MockIisa { + selected: vec![selected(new_idx)], + }, + queue.clone(), + MockChainClient, + events.clone(), + snapshot, + ); + + // Request size 1 == returned 1, so no shortfall event. + handle(ctx, &test_message(1)).await.expect("handler ok"); + + let captured = events.events(); + let proposed: Vec<_> = captured + .iter() + .filter_map(|e| match e { + CapturedEvent::Proposed { event, .. } => Some(event), + _ => None, + }) + .collect(); + assert_eq!(proposed.len(), 1, "exactly one Proposed: {captured:?}"); + assert_eq!( + proposed[0].candidates, + vec![new_idx.to_string()], + "proposed candidate is the new indexer" + ); + + assert!( + !captured + .iter() + .any(|e| matches!(e, CapturedEvent::Terminated { .. })), + "reassess must not emit terminated; the chain_listener sweep does: {captured:?}" + ); + + // The proposal was actually queued to the new indexer's URL. + assert_eq!(queue.proposals.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn never_accepted_unpaired_cancel_does_not_emit_terminated() { + // Both old agreements were never accepted on-chain (Created). One add + // pairs with the first old agreement; the second, unpaired old agreement + // reaches the cancel loop but, being never-accepted + // (`!needs_on_chain_cancel`), must NOT emit `terminated`. The add still + // emits `proposed`. Net: exactly one event, a `proposed`. + let new_idx = indexer_id(0x44); + let old_paired = indexer_id(0x55); + let old_unpaired = indexer_id(0x56); + + let mut snapshot = indexer_urls::Snapshot::new(); + snapshot.insert(new_idx, indexer_url()); + + let events = CapturingEventsProducer::new(); + let registry = MockRegistry { + active_agreements: vec![ + agreement(old_paired, IndexingAgreementStatus::Created), + agreement(old_unpaired, IndexingAgreementStatus::Created), + ], + accepted_count: 0, + chain_state_lookup_fails: false, + shortfall_active: std::sync::Mutex::new(false), + }; + let ctx = build_ctx( + registry, + MockIisa { + selected: vec![selected(new_idx)], + }, + MockQueue::default(), + MockChainClient, + events.clone(), + snapshot, + ); + + handle(ctx, &test_message(1)).await.expect("handler ok"); + + let captured = events.events(); + assert_eq!( + captured.len(), + 1, + "only Proposed; never-accepted unpaired cancel emits no Terminated: {captured:?}" + ); + assert!(matches!(captured[0], CapturedEvent::Proposed { .. })); + } +} + /// Olds reserved from cancellation: one per add-cancel pairing lost to a /// withheld addition. Olds beyond the pairing count are true surplus (IISA /// dropped them outright) and cancel immediately regardless of pacing. @@ -1084,6 +2190,7 @@ mod terms_hash_tests { .unwrap(), protocol_network: 1u64, chain_id: 1u64, + proposed_at: 0, }, } } diff --git a/bin/dipper-service/src/worker/handlers/send_indexing_agreement_proposal.rs b/bin/dipper-service/src/worker/handlers/send_indexing_agreement_proposal.rs index e4354e1b..4bb06e56 100644 --- a/bin/dipper-service/src/worker/handlers/send_indexing_agreement_proposal.rs +++ b/bin/dipper-service/src/worker/handlers/send_indexing_agreement_proposal.rs @@ -471,6 +471,13 @@ mod tests { Ok(vec![]) } + async fn count_accepted_agreements_by_deployment( + &self, + _deployment_id: &DeploymentId, + ) -> crate::registry::Result { + Ok(0) + } + async fn register_new_indexing_agreement( &self, _params: crate::registry::NewAgreementParams, @@ -880,6 +887,7 @@ mod tests { ), protocol_network: 1, chain_id: 1, + proposed_at: 0, }, }; IndexingAgreement { diff --git a/bin/dipper-service/src/worker/handlers/submit_offer.rs b/bin/dipper-service/src/worker/handlers/submit_offer.rs index 8f725a20..022dd07c 100644 --- a/bin/dipper-service/src/worker/handlers/submit_offer.rs +++ b/bin/dipper-service/src/worker/handlers/submit_offer.rs @@ -295,6 +295,7 @@ mod tests { ), protocol_network: 1, chain_id: 1, + proposed_at: 0, }, }; let nonce_uuid = uuid::Uuid::now_v7(); diff --git a/bin/dipper-service/src/worker/service.rs b/bin/dipper-service/src/worker/service.rs index fddabcaf..11c8e51d 100644 --- a/bin/dipper-service/src/worker/service.rs +++ b/bin/dipper-service/src/worker/service.rs @@ -72,6 +72,7 @@ where unresponsive_breaker, dips_accepting_cache, concurrency, + subgraph_indexing_agreements_events_emitter, } = state.into(); // A watch channel fans the stop signal out to every loop; a single mpsc @@ -105,6 +106,7 @@ where reassess_lock, unresponsive_breaker, dips_accepting_cache, + subgraph_indexing_agreements_events_emitter, }; let mut set = JoinSet::new(); @@ -253,7 +255,8 @@ where + AgreementRegistry + IndexerDenylistRegistry + PendingCancellationRegistry - + crate::network::service::chain_listener::ChainListenerStateRegistry, + + crate::network::service::chain_listener::ChainListenerStateRegistry + + Sync, W: WorkerQueue, C: IndexerClient, I: CandidateSelection, diff --git a/dipper-pgregistry/migrations/20260706000000_add_event_emission_tracking.sql b/dipper-pgregistry/migrations/20260706000000_add_event_emission_tracking.sql new file mode 100644 index 00000000..161fcfc4 --- /dev/null +++ b/dipper-pgregistry/migrations/20260706000000_add_event_emission_tracking.sql @@ -0,0 +1,58 @@ +-- Event-emission tracking for subgraph-indexing-agreement lifecycle events. +-- +-- The lifecycle events (accepted / terminated / expired) must not be lost on a +-- crash between the durable state change and the Kafka send. Rather than a +-- separate outbox table, we treat the agreement row itself as the durable +-- record: audit columns capture the event payload at reconcile time, and a +-- per-event "emitted_at" marker lets a sweep re-derive and re-send any event +-- whose marker is still NULL. Emission becomes self-healing / at-least-once. + +-- Audit columns: the on-chain payload observed at reconcile time. Timestamps are +-- chain seconds (BIGINT); tx hashes and canceled_by are stored as their 0x hex +-- string form (matching the event wire representation). All nullable: a row only +-- carries the fields relevant to the transitions it has been through. +ALTER TABLE dipper_reg_indexing_agreements + ADD COLUMN accepted_at BIGINT, + ADD COLUMN accepted_tx TEXT, + ADD COLUMN canceled_at BIGINT, + ADD COLUMN canceled_by TEXT, + ADD COLUMN canceled_tx TEXT; + +-- Emission markers: NULL until the corresponding event is confirmed sent to the +-- broker. A sweep emits for rows whose marker is NULL, then stamps it. +ALTER TABLE dipper_reg_indexing_agreements + ADD COLUMN accepted_event_emitted_at TIMESTAMPTZ, + ADD COLUMN terminated_event_emitted_at TIMESTAMPTZ, + ADD COLUMN expired_event_emitted_at TIMESTAMPTZ; + +-- Coverage-shortfall latch for the request. Drives the transition-based shortage +-- signal (fire once on entering shortfall, clear on recovery) instead of the +-- standing-condition emit that re-fired every reassessment. +ALTER TABLE dipper_reg_indexing_requests + ADD COLUMN shortfall_active BOOLEAN NOT NULL DEFAULT FALSE; + +-- Sweep-scan indexes. Partial, so they only cover rows still awaiting emission +-- (normally none), keeping them tiny. +-- +-- terminated: only agreements that were genuinely accepted on-chain +-- (accepted_at IS NOT NULL) are eligible -- a never-accepted agreement that was +-- canceled locally has nothing on-chain to have terminated. Statuses: +-- CanceledByRequester (3), CanceledByIndexer (4), AbandonedByIndexer (8). +CREATE INDEX idx_agreements_pending_terminated_event + ON dipper_reg_indexing_agreements (id) + WHERE status IN (3, 4, 8) + AND accepted_at IS NOT NULL + AND terminated_event_emitted_at IS NULL; + +-- accepted: rows whose accept was observed by the reconcile path (accepted_at IS +-- NOT NULL, so pre-feature rows are never backfilled) and not yet announced. NOT +-- gated on status: an agreement accepted-then-cancelled in one snapshot ends up +-- terminal with accepted_at set and must still emit its accepted event. +CREATE INDEX idx_agreements_pending_accepted_event + ON dipper_reg_indexing_agreements (id) + WHERE accepted_at IS NOT NULL AND accepted_event_emitted_at IS NULL; + +-- expired: Expired (5) rows not yet announced. +CREATE INDEX idx_agreements_pending_expired_event + ON dipper_reg_indexing_agreements (id) + WHERE status = 5 AND expired_event_emitted_at IS NULL; diff --git a/dipper-pgregistry/src/indexing_agreement.rs b/dipper-pgregistry/src/indexing_agreement.rs index ed909f59..53c20c32 100644 --- a/dipper-pgregistry/src/indexing_agreement.rs +++ b/dipper-pgregistry/src/indexing_agreement.rs @@ -291,6 +291,11 @@ pub struct TermsMetadata { pub protocol_network: ChainId, /// Indexed chain, e.g., `eip155:1` (Ethereum Mainnet). pub chain_id: ChainId, + + /// Unix-seconds proposal time, in the same clock as `Terms::deadline`. + /// `serde(default)` yields 0 for rows written before this field existed. + #[serde(default)] + pub proposed_at: u64, } /// The _indexing agreement_ [`fake`] implementation for test data generation. @@ -347,6 +352,7 @@ pub mod fake_impl { subgraph_deployment_id: DeploymentId::dummy_with_rng(config, rng), protocol_network: ChainId::dummy_with_rng(config, rng), chain_id: ChainId::dummy_with_rng(config, rng), + proposed_at: u64::dummy_with_rng(config, rng), } } } @@ -383,6 +389,7 @@ mod tests { .unwrap(), protocol_network: 42161, chain_id: 1, + proposed_at: 1_700_000_000, }, }; diff --git a/dipper-pgregistry/src/lib.rs b/dipper-pgregistry/src/lib.rs index f2566aa3..20b40ea6 100644 --- a/dipper-pgregistry/src/lib.rs +++ b/dipper-pgregistry/src/lib.rs @@ -20,8 +20,9 @@ pub use indexing_request::{ Status as IndexingRequestStatus, }; pub use postgres::{ - CancelKind, ChainListenerStateRow, NewAgreementParams, PgRegistry, ReconciliationItem, - ReconciliationOutcome, + CancelKind, ChainListenerStateRow, NewAgreementParams, PendingAcceptedEvent, + PendingExpiredEvent, PendingTerminatedEvent, PgRegistry, ReconciliationAudit, + ReconciliationItem, ReconciliationOutcome, }; pub use result::{Error, Result}; diff --git a/dipper-pgregistry/src/postgres.rs b/dipper-pgregistry/src/postgres.rs index bb96d0ad..1463989e 100644 --- a/dipper-pgregistry/src/postgres.rs +++ b/dipper-pgregistry/src/postgres.rs @@ -70,11 +70,152 @@ pub struct ReconciliationOutcome { } /// One row's reconciliation request inside a batched apply. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct ReconciliationItem { pub agreement_id: dipper_core::ids::IndexingAgreementId, pub apply_accept: bool, pub cancel: Option, + /// On-chain payload observed for this transition, persisted alongside the + /// status change so a later emission sweep can rebuild the lifecycle event + /// from the row alone. Only the fields relevant to the applied transition + /// are populated. + pub audit: ReconciliationAudit, +} + +/// Audit payload captured at reconcile time for later event emission. +/// +/// Timestamps are chain seconds; tx hashes and `canceled_by` are the 0x hex +/// string form matching the event wire representation. +#[derive(Debug, Clone, Default)] +pub struct ReconciliationAudit { + pub accepted_at: Option, + pub accepted_tx: Option, + pub canceled_at: Option, + pub canceled_by: Option, + pub canceled_tx: Option, +} + +/// A row that needs a `terminated` lifecycle event emitted. Sourced entirely +/// from the agreement row so the emission sweep is a pure DB -> broker drain. +#[derive(Debug, Clone)] +pub struct PendingTerminatedEvent { + pub agreement_id: IndexingAgreementId, + pub indexer_id: IndexerId, + pub deployment_id: DeploymentId, + pub protocol_network: ChainId, + /// Chain seconds of the on-chain cancel; `None` if the cancel audit was + /// never recorded (the sweep falls back to `updated_at`). + pub canceled_at: Option, + /// 0x address that cancelled; `None` -> the sweep falls back to the manager. + pub canceled_by: Option, + /// 0x hex of the cancel tx; `None` -> emitted empty. + pub canceled_tx: Option, + /// When the row was marked terminal; the `terminated_at` fallback. + pub updated_at: time::OffsetDateTime, +} + +impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for PendingTerminatedEvent { + fn from_row(row: &sqlx::postgres::PgRow) -> Result { + use sqlx::Row as _; + let agreement_id = row.try_get("id")?; + let PgIndexerId(indexer_id) = row.try_get("indexer_id")?; + let sqlx::types::Json(terms): sqlx::types::Json = + row.try_get("terms")?; + let canceled_at: Option = row.try_get("canceled_at")?; + let canceled_by: Option = row.try_get("canceled_by")?; + let canceled_tx: Option = row.try_get("canceled_tx")?; + let updated_at = row.try_get("updated_at")?; + Ok(Self { + agreement_id, + indexer_id, + deployment_id: terms.metadata.subgraph_deployment_id, + protocol_network: terms.metadata.protocol_network, + canceled_at: canceled_at.map(|v| v as u64), + canceled_by, + canceled_tx, + updated_at, + }) + } +} + +/// A row that needs an `accepted` lifecycle event emitted. Sourced entirely from +/// the agreement row so the emission sweep is a pure DB -> broker drain. +#[derive(Debug, Clone)] +pub struct PendingAcceptedEvent { + pub agreement_id: IndexingAgreementId, + pub indexer_id: IndexerId, + pub deployment_id: DeploymentId, + pub protocol_network: ChainId, + /// Chain seconds of the on-chain accept (always present -- the query gates on + /// `accepted_at IS NOT NULL`). + pub accepted_at: u64, + /// 0x hex of the accept tx; empty if unset. + pub accepted_tx: String, + /// Agreement end timestamp, from the row's terms. + pub ends_at: u64, + /// Payer address, from the row's terms. + pub payer: Address, +} + +impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for PendingAcceptedEvent { + fn from_row(row: &sqlx::postgres::PgRow) -> Result { + use sqlx::Row as _; + let agreement_id = row.try_get("id")?; + let PgIndexerId(indexer_id) = row.try_get("indexer_id")?; + let sqlx::types::Json(terms): sqlx::types::Json = + row.try_get("terms")?; + let accepted_at: i64 = row.try_get("accepted_at")?; + let accepted_tx: Option = row.try_get("accepted_tx")?; + Ok(Self { + agreement_id, + indexer_id, + deployment_id: terms.metadata.subgraph_deployment_id, + protocol_network: terms.metadata.protocol_network, + accepted_at: accepted_at as u64, + accepted_tx: accepted_tx.unwrap_or_default(), + ends_at: terms.ends_at, + payer: terms.payer, + }) + } +} + +/// A row that needs a `request.expired` lifecycle event emitted. Sourced from +/// the agreement row alone; `request_expired_at` is the terms deadline (the true +/// expiry instant), so the sweep needs no chain-time snapshot. +#[derive(Debug, Clone)] +pub struct PendingExpiredEvent { + pub agreement_id: IndexingAgreementId, + pub indexer_id: IndexerId, + pub deployment_id: DeploymentId, + pub protocol_network: ChainId, + /// Proposal time (terms `proposed_at`, or the row `created_at` for + /// pre-`proposed_at` rows). + pub request_proposed_at: u64, + /// The deadline that passed -- the true expiry instant. + pub request_expired_at: u64, +} + +impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for PendingExpiredEvent { + fn from_row(row: &sqlx::postgres::PgRow) -> Result { + use sqlx::Row as _; + let agreement_id = row.try_get("id")?; + let PgIndexerId(indexer_id) = row.try_get("indexer_id")?; + let created_at: time::OffsetDateTime = row.try_get("created_at")?; + let sqlx::types::Json(terms): sqlx::types::Json = + row.try_get("terms")?; + let request_proposed_at = match terms.metadata.proposed_at { + 0 => created_at.unix_timestamp().max(0) as u64, + ts => ts, + }; + Ok(Self { + agreement_id, + indexer_id, + deployment_id: terms.metadata.subgraph_deployment_id, + protocol_network: terms.metadata.protocol_network, + request_proposed_at, + request_expired_at: terms.deadline, + }) + } } /// A registry that stores indexing requests and agreements in a PostgreSQL database. @@ -296,6 +437,29 @@ impl PgRegistry { .map_err(Into::into) } + /// Counts the indexing agreements for a deployment that are accepted on-chain. + /// + /// Used to populate `remaining_accepted_indexing_agreements` on the + /// `terminated` lifecycle event: the number of still-active accepted + /// agreements left on the Subgraph (deployment) after a cancellation. + pub async fn count_accepted_agreements_by_deployment( + &self, + deployment_id: &DeploymentId, + ) -> Result { + let (count,): (i64,) = sqlx::query_as( + r#" + SELECT COUNT(*) + FROM dipper_reg_indexing_agreements + WHERE deployment_id = $1 AND status = $2 + "#, + ) + .bind(PgDeploymentId(*deployment_id)) + .bind(IndexingAgreementStatus::AcceptedOnChain) + .fetch_one(&self.pool) + .await?; + Ok(count) + } + pub async fn register_new_indexing_agreement( &self, params: NewAgreementParams, @@ -955,11 +1119,219 @@ impl PgRegistry { outcomes.entry(id).or_default().did_cancel = true; } + // Persist the observed on-chain payload for every row that actually + // transitioned, in the same transaction as the status change, so a later + // emission sweep can rebuild the lifecycle event from the row alone. + for item in items { + let outcome = outcomes + .get(&item.agreement_id) + .copied() + .unwrap_or_default(); + if outcome.did_accept { + persist_accept_audit(&mut tx, &item.agreement_id, &item.audit).await?; + } + if outcome.did_cancel { + persist_cancel_audit(&mut tx, &item.agreement_id, &item.audit).await?; + } + } + tx.commit().await?; Ok(outcomes) } + /// Fetch a batch of agreements awaiting a `terminated` event: in a + /// terminal-cancel state, genuinely accepted on-chain (`accepted_at IS NOT + /// NULL`, so a never-accepted local cancel is excluded), and not yet + /// emitted. Oldest-marked first so the backlog drains in order. + pub async fn get_agreements_pending_terminated_emission( + &self, + limit: i64, + ) -> Result, Error> { + let rows = sqlx::query_as::<_, PendingTerminatedEvent>( + r#" + SELECT id, indexer_id, terms, canceled_at, canceled_by, canceled_tx, updated_at + FROM dipper_reg_indexing_agreements + WHERE status IN ($1, $2, $3) + AND accepted_at IS NOT NULL + AND terminated_event_emitted_at IS NULL + ORDER BY updated_at ASC + LIMIT $4 + "#, + ) + .bind(IndexingAgreementStatus::CanceledByRequester) + .bind(IndexingAgreementStatus::CanceledByIndexer) + .bind(IndexingAgreementStatus::AbandonedByIndexer) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows) + } + + /// Stamp the `terminated` emission marker after a confirmed broker send. + pub async fn mark_terminated_event_emitted( + &self, + agreement_id: &IndexingAgreementId, + ) -> Result<(), Error> { + sqlx::query( + r#" + UPDATE dipper_reg_indexing_agreements + SET terminated_event_emitted_at = timezone('UTC', now()) + WHERE id = $1 + "#, + ) + .bind(agreement_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Fetch a batch of agreements awaiting an `accepted` event: the accept was + /// observed by the reconcile path (`accepted_at IS NOT NULL`, so pre-feature + /// rows are never backfilled) and not yet emitted. Note this is NOT gated on + /// current status: an agreement accepted and then cancelled in a single + /// snapshot (dipper was behind) ends up `CanceledBy*` with `accepted_at` set, + /// and must still emit its `accepted` before the `terminated`. Oldest-accepted + /// first. + pub async fn get_agreements_pending_accepted_emission( + &self, + limit: i64, + ) -> Result, Error> { + let rows = sqlx::query_as::<_, PendingAcceptedEvent>( + r#" + SELECT id, indexer_id, terms, accepted_at, accepted_tx + FROM dipper_reg_indexing_agreements + WHERE accepted_at IS NOT NULL + AND accepted_event_emitted_at IS NULL + ORDER BY accepted_at ASC + LIMIT $1 + "#, + ) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows) + } + + /// Stamp the `accepted` emission marker after a confirmed broker send. + pub async fn mark_accepted_event_emitted( + &self, + agreement_id: &IndexingAgreementId, + ) -> Result<(), Error> { + sqlx::query( + r#" + UPDATE dipper_reg_indexing_agreements + SET accepted_event_emitted_at = timezone('UTC', now()) + WHERE id = $1 + "#, + ) + .bind(agreement_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Fetch a batch of agreements awaiting a `request.expired` event: currently + /// `Expired` and not yet emitted. A row recovered `Expired -> AcceptedOnChain` + /// before the sweep runs is no longer selected, so no premature `expired` + /// contradicts a later `accepted`. Oldest-updated first. + pub async fn get_agreements_pending_expired_emission( + &self, + limit: i64, + ) -> Result, Error> { + let rows = sqlx::query_as::<_, PendingExpiredEvent>( + r#" + SELECT id, indexer_id, created_at, terms + FROM dipper_reg_indexing_agreements + WHERE status = $1 + AND expired_event_emitted_at IS NULL + ORDER BY updated_at ASC + LIMIT $2 + "#, + ) + .bind(IndexingAgreementStatus::Expired) + .bind(limit) + .fetch_all(&self.pool) + .await?; + Ok(rows) + } + + /// Stamp the `request.expired` emission marker after a confirmed broker send. + pub async fn mark_expired_event_emitted( + &self, + agreement_id: &IndexingAgreementId, + ) -> Result<(), Error> { + sqlx::query( + r#" + UPDATE dipper_reg_indexing_agreements + SET expired_event_emitted_at = timezone('UTC', now()) + WHERE id = $1 + "#, + ) + .bind(agreement_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Record the accepted audit payload out-of-band from `apply_reconciliation`. + /// + /// Used for the rejected-then-accepted anomaly: such a row goes + /// `Rejected -> Canceled` without ever transiting `AcceptedOnChain`, so the + /// normal accept transition never records it. Persisting it here (from the + /// observed snapshot) marks the row as genuinely accepted so its eventual + /// `terminated` is sweep-eligible. `COALESCE` keeps any existing value. + pub async fn record_accepted_audit( + &self, + agreement_id: &IndexingAgreementId, + accepted_at: u64, + accepted_tx: &str, + ) -> Result<(), Error> { + sqlx::query( + r#" + UPDATE dipper_reg_indexing_agreements + SET accepted_at = COALESCE(accepted_at, $2), + accepted_tx = COALESCE(accepted_tx, $3) + WHERE id = $1 + "#, + ) + .bind(agreement_id) + .bind(accepted_at as i64) + .bind(accepted_tx) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Record the cancel audit payload for a dipper-initiated cancel so the + /// emission sweep can populate the `terminated` event's tx/by/at fields. + /// `COALESCE` keeps any value already observed on-chain. Best-effort + /// enrichment: the event still emits (with fallbacks) if never recorded. + pub async fn record_cancel_audit( + &self, + agreement_id: &IndexingAgreementId, + canceled_at: u64, + canceled_by: &str, + canceled_tx: Option<&str>, + ) -> Result<(), Error> { + sqlx::query( + r#" + UPDATE dipper_reg_indexing_agreements + SET canceled_at = COALESCE(canceled_at, $2), + canceled_by = COALESCE(canceled_by, $3), + canceled_tx = COALESCE(canceled_tx, $4) + WHERE id = $1 + "#, + ) + .bind(agreement_id) + .bind(canceled_at as i64) + .bind(canceled_by) + .bind(canceled_tx) + .execute(&self.pool) + .await?; + Ok(()) + } + // ========================================================================= // Reassignment operations // ========================================================================= @@ -1003,6 +1375,30 @@ impl PgRegistry { .map_err(Into::into) } + /// Atomically set the coverage-shortfall latch on a request, returning whether + /// the value changed. The `shortfall_active <> $2` guard makes the UPDATE a + /// no-op (returning no row) when the value is unchanged, so the caller learns + /// whether this was a transition. + pub async fn set_indexing_request_shortfall_active( + &self, + id: &IndexingRequestId, + active: bool, + ) -> Result { + let record: Option<(IndexingRequestId,)> = sqlx::query_as( + r#" + UPDATE dipper_reg_indexing_requests + SET shortfall_active = $2 + WHERE id = $1 AND shortfall_active <> $2 + RETURNING id + "#, + ) + .bind(id) + .bind(active) + .fetch_optional(&self.pool) + .await?; + Ok(record.is_some()) + } + // ========================================================================= // Deadline expiration operations // ========================================================================= @@ -1726,6 +2122,60 @@ async fn batch_update_status_from( /// Transition an agreement's status inside a transaction. Thin wrapper /// over `batch_update_status_from` for the single-row case. +/// Persist the accepted-transition audit payload for a single row. `COALESCE` +/// keeps any already-stored value rather than clobbering it with `NULL`. +async fn persist_accept_audit( + tx: &mut sqlx::Transaction<'_, Postgres>, + id: &IndexingAgreementId, + audit: &ReconciliationAudit, +) -> Result<(), Error> { + if audit.accepted_at.is_none() && audit.accepted_tx.is_none() { + return Ok(()); + } + sqlx::query( + r#" + UPDATE dipper_reg_indexing_agreements + SET accepted_at = COALESCE($2, accepted_at), + accepted_tx = COALESCE($3, accepted_tx) + WHERE id = $1 + "#, + ) + .bind(id) + .bind(audit.accepted_at.map(|v| v as i64)) + .bind(audit.accepted_tx.as_deref()) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Persist the cancel-transition audit payload for a single row. `COALESCE` +/// keeps any already-stored value rather than clobbering it with `NULL`. +async fn persist_cancel_audit( + tx: &mut sqlx::Transaction<'_, Postgres>, + id: &IndexingAgreementId, + audit: &ReconciliationAudit, +) -> Result<(), Error> { + if audit.canceled_at.is_none() && audit.canceled_by.is_none() && audit.canceled_tx.is_none() { + return Ok(()); + } + sqlx::query( + r#" + UPDATE dipper_reg_indexing_agreements + SET canceled_at = COALESCE($2, canceled_at), + canceled_by = COALESCE($3, canceled_by), + canceled_tx = COALESCE($4, canceled_tx) + WHERE id = $1 + "#, + ) + .bind(id) + .bind(audit.canceled_at.map(|v| v as i64)) + .bind(audit.canceled_by.as_deref()) + .bind(audit.canceled_tx.as_deref()) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn update_status_from( tx: &mut sqlx::Transaction<'_, Postgres>, agreement_id: &IndexingAgreementId, diff --git a/dipper-pgregistry/tests/it_registry_postgres.rs b/dipper-pgregistry/tests/it_registry_postgres.rs index ecf29dd9..1cd628ed 100644 --- a/dipper-pgregistry/tests/it_registry_postgres.rs +++ b/dipper-pgregistry/tests/it_registry_postgres.rs @@ -5,7 +5,7 @@ use std::collections::HashSet; use dipper_core::ids::{IndexingAgreementId, IndexingRequestId}; use dipper_pgregistry::{ CancelKind, Error, IndexingAgreementStatus, IndexingAgreementTerms, IndexingRequestStatus, - NewAgreementParams, PgRegistry, ReconciliationItem, + NewAgreementParams, PgRegistry, ReconciliationAudit, ReconciliationItem, }; use fake::{Fake, Faker}; use pgtemp::PgTempDB; @@ -2655,24 +2655,28 @@ async fn apply_reconciliation_batch_handles_all_four_item_shapes() { agreement_id: accept_only_id, apply_accept: true, cancel: None, + audit: ReconciliationAudit::default(), }, // cancel-only by indexer: AcceptedOnChain -> CanceledByIndexer ReconciliationItem { agreement_id: cancel_by_indexer_id, apply_accept: false, cancel: Some(CancelKind::ByIndexer), + audit: ReconciliationAudit::default(), }, // paired accept+cancel-by-requester: Created -> AcceptedOnChain -> CanceledByRequester ReconciliationItem { agreement_id: paired_id, apply_accept: true, cancel: Some(CancelKind::ByRequester), + audit: ReconciliationAudit::default(), }, // accept-only recovery from Expired ReconciliationItem { agreement_id: recover_expired_id, apply_accept: true, cancel: None, + audit: ReconciliationAudit::default(), }, ]; @@ -2743,6 +2747,98 @@ async fn apply_reconciliation_batch_handles_all_four_item_shapes() { ); } +#[tokio::test] +async fn pending_emission_queries_cover_the_paired_accept_then_cancel() { + // Regression guard: an agreement accepted and then cancelled in a single + // snapshot (paired accept+cancel -- dipper was behind) ends up terminal, but + // must still surface for BOTH the accepted and terminated emission sweeps. The + // accepted event must not be lost just because the row is no longer + // AcceptedOnChain. + let (db, _temp_db) = temp_registry_db().await; + run_fixture( + &db, + include_str!("fixtures/0003_multi_indexer_agreements.sql"), + ) + .await + .expect("Failed to run fixture"); + let registry = PgRegistry::new(db); + + let paired_id = + IndexingAgreementId::from_bytes([0xbb, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]); + + // Paired accept+cancel with the on-chain audit populated, as the chain_listener + // does from the observed snapshot. + let items = vec![ReconciliationItem { + agreement_id: paired_id, + apply_accept: true, + cancel: Some(CancelKind::ByRequester), + audit: ReconciliationAudit { + accepted_at: Some(1_700_000_001), + accepted_tx: Some("0xacc".to_string()), + canceled_at: Some(1_700_000_002), + canceled_by: Some("0xcxl".to_string()), + canceled_tx: Some("0xcxltx".to_string()), + }, + }]; + let outcomes = registry + .apply_reconciliation_batch(&items) + .await + .expect("batch should succeed"); + let paired = outcomes.get(&paired_id).copied().unwrap_or_default(); + assert!( + paired.did_accept && paired.did_cancel, + "paired accept+cancel applied" + ); + + // The row is now CanceledByRequester, yet it must appear in BOTH pending sets. + let pending_accepted = registry + .get_agreements_pending_accepted_emission(100) + .await + .expect("accepted query"); + assert!( + pending_accepted.iter().any(|p| p.agreement_id == paired_id), + "accepted event must NOT be lost for an accepted-then-cancelled agreement" + ); + let pending_terminated = registry + .get_agreements_pending_terminated_emission(100) + .await + .expect("terminated query"); + assert!( + pending_terminated + .iter() + .any(|p| p.agreement_id == paired_id), + "terminated event must surface for the cancelled agreement" + ); + + // After stamping each marker, neither query returns the row again (emit-once). + registry + .mark_accepted_event_emitted(&paired_id) + .await + .expect("mark accepted"); + registry + .mark_terminated_event_emitted(&paired_id) + .await + .expect("mark terminated"); + assert!( + !registry + .get_agreements_pending_accepted_emission(100) + .await + .expect("accepted query") + .iter() + .any(|p| p.agreement_id == paired_id), + "accepted marker prevents re-emit" + ); + assert!( + !registry + .get_agreements_pending_terminated_emission(100) + .await + .expect("terminated query") + .iter() + .any(|p| p.agreement_id == paired_id), + "terminated marker prevents re-emit" + ); +} + #[tokio::test] async fn apply_reconciliation_batch_no_op_cas_miss_present_with_default_outcome() { // Documents the contract the chain_listener relies on to distinguish @@ -2769,6 +2865,7 @@ async fn apply_reconciliation_batch_no_op_cas_miss_present_with_default_outcome( agreement_id: already_canceled_id, apply_accept: false, cancel: Some(CancelKind::ByIndexer), + audit: ReconciliationAudit::default(), }]; //* When @@ -2826,6 +2923,7 @@ async fn apply_reconciliation_batch_paired_rolls_back_on_unmatched_cancel() { agreement_id: created_id, apply_accept: true, cancel: Some(CancelKind::ByIndexer), + audit: ReconciliationAudit::default(), }]; //* When diff --git a/dipper-producer/Cargo.toml b/dipper-producer/Cargo.toml new file mode 100644 index 00000000..489f170b --- /dev/null +++ b/dipper-producer/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "dipper-producer" +description = "Provides a shareable message producer layer" +version = "0.1.0" +edition = "2024" + +[dependencies] +async-trait.workspace = true +chrono = "0.4.45" +fs-err = "3.3.0" +prost = "0.14.4" +rskafka = { version = "0.6.0", features = ["transport-tls"] } +rustls = "0.23.41" +rustls-pemfile = "2.2.0" +serde.workspace = true +thegraph-core.workspace = true +thiserror.workspace = true +tokio.workspace = true +tracing.workspace = true +uuid = { workspace = true, features = ["v7"] } +webpki-roots = "1.0.8" + +# Protobuf code generation dependencies +# Run with: RUSTFLAGS="--cfg gen_event_proto" cargo check -p dipper-producer +[target.'cfg(gen_event_proto)'.build-dependencies] +prost-build = "0.14.4" + +[lints.rust] +# Allow the gen_event_proto cfg flag used for conditional protobuf code generation +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(gen_event_proto)'] } + +[build-dependencies] +prost-build = "0.14.4" \ No newline at end of file diff --git a/dipper-producer/README.md b/dipper-producer/README.md new file mode 100644 index 00000000..32ee5100 --- /dev/null +++ b/dipper-producer/README.md @@ -0,0 +1,22 @@ +# dipper-producer + +This crate contains protobuf definitions for dipper indexer agreement event streaming. +The generated Rust bindings are committed to the repository and only need to be regenerated when the `.proto` file changes. + +## Protobuf Generation + +The build script uses a configuration flag `gen_event_proto` that enables protobuf code generation via `prost-build`. When enabled, the build script compiles `proto/indexing-agreement-events.proto` into Rust types under `src/proto/`. + +To regenerate protobuf bindings, run: + +```bash +just gen-indexing-agreement-events-proto +``` + +Or using the full `cargo` command: + +```bash +RUSTFLAGS="--cfg gen_event_proto" cargo check -p dipper-producer +``` + +This will regenerate `src/proto/dipper.subgraph.indexing.agreement.events.v1.rs` from `proto/indexing-agreement-events.proto`. diff --git a/dipper-producer/build.rs b/dipper-producer/build.rs new file mode 100644 index 00000000..c894d68b --- /dev/null +++ b/dipper-producer/build.rs @@ -0,0 +1,21 @@ +fn main() -> Result<(), Box> { + #[cfg(gen_event_proto)] + { + println!("cargo:warning=Config 'gen_event_proto' enabled: Running protobuf codegen"); + + let mut config = prost_build::Config::new(); + config.out_dir("src/proto"); + config.protoc_arg("--experimental_allow_proto3_optional"); + + config.compile_protos(&["proto/indexing-agreement-events.proto"], &["proto/"])?; + + // Instruct cargo to rerun this build script if any of the proto files change + println!("cargo:rerun-if-changed=proto"); + } + #[cfg(not(gen_event_proto))] + { + println!("cargo:debug=Config 'gen_event_proto' not enabled: Skipping protobuf codegen"); + } + + Ok(()) +} diff --git a/dipper-producer/proto/indexing-agreement-events.proto b/dipper-producer/proto/indexing-agreement-events.proto new file mode 100644 index 00000000..1ef30288 --- /dev/null +++ b/dipper-producer/proto/indexing-agreement-events.proto @@ -0,0 +1,177 @@ +// Indexing Agreements Events Protocol Buffer Schema +// +// This schema defines the message body and event types emitted during different event outcomes with indexer agreements +// through the DIPS ecosystem upgrades to The Graph protocol. +// When a Subgraph developer submits a Subgraph for 1->many indexing agreements, this emits events through this lifecycle. +// +// Events are published to Kafka for realtime monitoring by downstream systems/consumers. +// +// Event flow: +// 1. subgraph.indexing.agreement.request.received - Emitted when the Subgraph dev sends an event to begin looking for indexing agreements for their Subgraph +// 2. subgraph.indexing.agreement.proposed - Emitted once dipper has gone to IISA, gotten candidates, and sent the gRPC offers, but before anyone has accepted on chain +// 3. subgraph.indexing.agreement.accepted - Emitted when an Indexer agrees to the indexing request for the Subgraph +// 4. subgraph.indexing.agreement.request.expired - Emitted when an indexer fails to accept within the time limit +// 5. subgraph.indexing.agreement.n_indexers_unavailable - Emitted when selection returns fewer candidates than requested +// 6. subgraph.indexing.agreement.terminated - Emitted if either the Subgraph developer, or Indexer cancel an indexing agreement on a Subgraph. +// +// Partition key format: {the_graph_network}/{subgraph_deployment_qm_hash} +// where the_graph_network is the CAIP-2 chain id of The Graph protocol network the Subgraph is +// published to. Example: eip155:42161/QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9 + +syntax = "proto3"; + +package dipper.subgraph.indexing.agreement.events.v1; + +// SubgraphIndexingAgreementEvent is the envelope that wraps all subgraph indexing agreement event types. +// +// Provides: +// - event_id -> unique identifier (uuid) +// - event_type -> event type discrimination +// - event_version -> versioning for schema evolution awareness to the consumer +// - timestamp -> when the event occurred, useful for ordering/debugging +// - subgraph_deployment_qm_hash -> Qm hash of the Subgraph deployment +// - the_graph_network -> CAIP-2 chain id of The Graph protocol network the Subgraph Deployment is published to (e.g. eip155:42161) +// - payload -> the event payload, determined by the event_type +message SubgraphIndexingAgreementEvent { + // Unique event identifier. + // Format: UUID v7 (time-ordered) for natural chronological sorting. + // Example: "01912345-6789-7abc-def0-123456789abc" + string event_id = 1; + + // Event type discriminator for routing and filtering. + // Values: "subgraph.indexing.agreement.request.received", "subgraph.indexing.agreement.proposed", "subgraph.indexing.agreement.accepted", "subgraph.indexing.agreement.request.expired", "subgraph.indexing.agreement.n_indexers_unavailable", "subgraph.indexing.agreement.terminated" + string event_type = 2; + + // Schema version for forward compatibility. + // Consumers should handle unknown fields gracefully. + // Current version: "1.0" + string event_version = 3; + + // Event timestamp in RFC 3339 format. + // Example: "2024-01-15T10:30:00.123Z" + string timestamp = 4; + + // Qm hash of the Subgraph deployment with a submitted indexing agreement event. + string subgraph_deployment_qm_hash = 5; + + // CAIP-2 chain id of The Graph protocol network where the Subgraph was published to. + // Format: "eip155:{chain_id}". Examples: "eip155:42161" (arbitrum), "eip155:421614" (arbitrum-sepolia) + string the_graph_network = 6; + + // Event payload - exactly one of the specific event types. + // Use event_type field to determine which payload is present. + oneof payload { + SubgraphIndexingAgreementRequestReceived subgraph_indexing_agreement_request_received = 7; + SubgraphIndexingAgreementProposed subgraph_indexing_agreement_proposed = 8; + SubgraphIndexingAgreementAccepted subgraph_indexing_agreement_accepted = 9; + SubgraphIndexingAgreementRequestExpired subgraph_indexing_agreement_request_expired = 10; + SubgraphIndexingAgreementNIndexersUnavailable subgraph_indexing_agreement_n_indexers_unavailable = 11; + SubgraphIndexingAgreementTerminated subgraph_indexing_agreement_terminated = 12; + } +} + +// SubgraphIndexingAgreementRequestReceived is emitted when a Subgraph developer submits a request to the dipper to find +// indexers to index the given Subgraph. +// +// Event type: subgraph.indexing.agreement.request.received +message SubgraphIndexingAgreementRequestReceived { + // Number of indexing agreements requested by the Subgraph developer. + int32 agreements_requested = 1; +} + +// SubgraphIndexingAgreementProposed is emitted once the dipper has gone to the IISA, received candidates (indexers), +// sent gRPC offers to these candidates, but before any candidate has accepted onchain +// +// Event type: subgraph.indexing.agreement.proposed +message SubgraphIndexingAgreementProposed { + // 0x addresses of the Indexer candidates dipper sent agreement offers to in + // this proposal round (the IISA-selected indexers newly proposed this cycle). + repeated string candidates = 1; + + // Timestamp of when the request expires + uint64 request_expires_at = 2; +} + +// SubgraphIndexingAgreementAccepted is emitted when an Indexer candidate has accepted onchain the Subgraph indexing request. +// +// Event type: subgraph.indexing.agreement.accepted +message SubgraphIndexingAgreementAccepted { + // 0x hex of the on-chain agreement id (bytes16) that was accepted. + // Stable, unique per agreement -- consumers dedup on (agreement_id, event_type) + // since emission is at-least-once (an accepted event may be redelivered). + string agreement_id = 6; + + // 0x address of the Indexer who accepted the indexing agreement + string indexer = 1; + + // Timestamp of when the agreement was accepted + uint64 accepted_at = 2; + + // 0x Hex of the onchain transaction for the accepted indexing agreement + string accepted_tx = 3; + + // Timestamp of when the agreement is scheduled to end + uint64 ends_at = 4; + + // 0x address of the payer + string payer = 5; +} + +// SubgraphIndexingAgreementRequestExpired is emitted when an Indexer candidate fails to accept within the provided time limit +// +// Event type: subgraph.indexing.agreement.request.expired +message SubgraphIndexingAgreementRequestExpired { + // 0x hex of the on-chain agreement id (bytes16) that expired. + // Stable, unique per agreement -- consumers dedup on (agreement_id, event_type) + // since emission is at-least-once (an expired event may be redelivered). + string agreement_id = 4; + + // 0x address of the Indexer who failed to accept the agreement within the given time limit + string indexer = 1; + + // Timestamp of when the Indexing agreement was proposed + uint64 request_proposed_at = 2; + + // Timestamp of when the required expired + uint64 request_expired_at = 3; +} + +// SubgraphIndexingAgreementNIndexersUnavailable is emitted when the IISA selection returns fewer Indexer candidates than requested +// +// Event type: subgraph.indexing.agreement.n_indexers_unavailable +message SubgraphIndexingAgreementNIndexersUnavailable { + // Number of indexing agreements requested by the Subgraph developer. + int32 agreements_requested = 1; + + // Number of indexer candidates selected by the IISA + int32 candidates_returned = 2; +} + +// SubgraphIndexingAgreementTerminated is emitted if either the Subgraph developer, or Indexer, cancel the agreement +// +// Event type: subgraph.indexing.agreement.terminated +message SubgraphIndexingAgreementTerminated { + // 0x hex of the on-chain agreement id (bytes16) that was terminated. + // Stable, unique per agreement -- consumers dedup on (agreement_id, event_type) + // since emission is at-least-once (a terminated event may be redelivered). + string agreement_id = 6; + + // 0x address of the Indexer that had the accepted agreement that was terminated + string indexer = 1; + + // Timestamp of when the Indexing Agreement was terminated + uint64 terminated_at = 2; + + // 0x address of the entity (developer, indexer, etc) that terminated the agreement + string terminated_by = 3; + + // 0x hex of the onchain transaction of the agreement cancellation + string terminated_tx = 4; + + // Count of remaining active (accepted) agreements on the Subgraph, excluding + // the one just terminated. + // 0 -> the Subgraph no longer has any active, accepted indexing agreements. + // -1 -> the count was unavailable when the event was emitted (e.g. a transient + // datastore error); treat as unknown, not as zero. + int32 remaining_accepted_indexing_agreements = 5; +} \ No newline at end of file diff --git a/dipper-producer/src/events.rs b/dipper-producer/src/events.rs new file mode 100644 index 00000000..9dd67901 --- /dev/null +++ b/dipper-producer/src/events.rs @@ -0,0 +1,69 @@ +//! Subgraph Indexing agreement event streaming module. +//! +//! This module provides infrastructure for emitting Subgraph Indexing agreement lifecycle events: +//! - subgraph.indexing.agreement.request.receive +//! - subgraph.indexing.agreement.proposed +//! - subgraph.indexing.agreement.accepted +//! - subgraph.indexing.agreement.request.expired +//! - subgraph.indexing.agreement.n_indexers_unavailable +//! - subgraph.indexing.agreement.terminated +//! +//! # Example - disabled +//! +//! ```ignore +//! use dipper_producer::events::SubgraphIndexingAgreementsEventsEmitter; +//! +//! SubgraphIndexingAgreementsEventsEmitter::disabled(); +//! ``` +//! +//! # Example - enabled with Kafka config +//! +//! ```ignore +//! use dipper_producer::{ +//! events::SubgraphIndexingAgreementsEventsEmitter, +//! kafka::{KafkaConfig, KafkaProducer, proto} +//! }; +//! +//! let config = KafkaConfig { +//! brokers: vec!["localhost:9092".to_string()], +//! topic: "dipper.subgraph.indexing.agreement.events".to_string(), +//! partitions: 16, +//! }; +//! +//! let producer = match tokio::time::timeout( +//! std::time::Duration::from_secs(30), +//! KafkaProducer::new(&config), +//! ) +//! .await +//! { +//! Ok(Ok(producer)) => producer, +//! Ok(Err(err)) => { +//! tracing::warn!(error = %err, "failed to initialize KafkaProducer, events disabled"); +//! return std::sync::Arc::new(SubgraphIndexingAgreementsEventsEmitter::disabled()); +//! }, +//! Err(elapsed) => { +//! tracing::warn!(error = %elapsed, "failed to initialize KafkaProducer, events disabled"); +//! return std::sync::Arc::new(SubgraphIndexingAgreementsEventsEmitter::disabled()); +//! } +//! }; +//! +//! let emitter = std::sync::Arc::new(SubgraphIndexingAgreementsEventsEmitter::enabled( +//! std::sync::Arc::new(producer), +//! 16, +//! )); +//! +//! emitter.produce_subgraph_indexing_agreement_request_received( +//! "QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9".parse().unwrap(), +//! 42161, // protocol network chain id; rendered to "eip155:42161" on the wire +//! proto::SubgraphIndexingAgreementRequestReceived { +//! agreements_requested: 2 +//! } +//! ); +//! ``` + +mod subgraph_indexing_agreements_events_emitter; + +pub use subgraph_indexing_agreements_events_emitter::{ + Caip2ChainId, EmitError, SubgraphIndexingAgreementEventsProducer, + SubgraphIndexingAgreementsEventsEmitter, +}; diff --git a/dipper-producer/src/events/subgraph_indexing_agreements_events_emitter.rs b/dipper-producer/src/events/subgraph_indexing_agreements_events_emitter.rs new file mode 100644 index 00000000..5437db74 --- /dev/null +++ b/dipper-producer/src/events/subgraph_indexing_agreements_events_emitter.rs @@ -0,0 +1,898 @@ +//! Subgraph Indexing Agreement event emitter to Kafka topic utilities + +use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use async_trait::async_trait; +use prost::Message; +use thegraph_core::{DeploymentId, alloy::primitives::ChainId}; +use tokio::{ + sync::{Notify, mpsc}, + task::JoinHandle, +}; + +use crate::{ + kafka::{KafkaConfig, KafkaProducer}, + proto, +}; + +/// Failure of a durable-tier emit (see [`SubgraphIndexingAgreementEventsProducer`]). +/// +/// The caller must treat this as "not yet emitted" and leave its DB marker +/// unset so the event is retried on the next sweep. +#[derive(Debug, thiserror::Error)] +pub enum EmitError { + #[error("failed to send lifecycle event to broker: {0}")] + Send(String), +} + +/// CAIP-2 identifier for an EVM (`eip155`) chain. +/// +/// Wraps a numeric [`ChainId`] and renders to its CAIP-2 string form +/// `eip155:{chain_id}` (e.g. `eip155:42161`) when written to the wire. Construct +/// one from a chain id via [`From`] / `ChainId::into`, so call sites pass the raw +/// chain id without repeating the `eip155:` formatting. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Caip2ChainId(ChainId); + +impl From for Caip2ChainId { + fn from(chain_id: ChainId) -> Self { + Self(chain_id) + } +} + +impl std::fmt::Display for Caip2ChainId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "eip155:{}", self.0) + } +} + +/// Produces Subgraph Indexing Agreement lifecycle events. +/// +/// Abstracts [`SubgraphIndexingAgreementsEventsEmitter`] so callers can depend on +/// the behavior rather than the concrete Kafka emitter, and tests can substitute a +/// capturing double. `the_graph_network` is the protocol network's numeric +/// [`ChainId`]; implementors render it to its CAIP-2 form (`eip155:{id}`) on the wire. +/// +/// # Two delivery tiers +/// +/// - **Diagnostic** (`request.received`, `proposed`, `n_indexers_unavailable`): +/// the `produce_*` methods enqueue onto a bounded in-memory channel and return +/// immediately. Best-effort; dropped on a full/closed queue. +/// - **Durable** (`terminated`, and later `accepted` / `expired`): the `emit_*` +/// methods send synchronously and report success, so the caller persists a DB +/// emission marker only after the broker confirms the send. A failure leaves +/// the marker unset and the event is re-derived and retried on the next sweep. +#[async_trait] +pub trait SubgraphIndexingAgreementEventsProducer: Send + Sync { + /// Produces a subgraph.indexing.agreement.request.received event + fn produce_subgraph_indexing_agreement_request_received( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementRequestReceived, + ); + + /// Produces a subgraph.indexing.agreement.proposed event + fn produce_subgraph_indexing_agreement_proposed( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementProposed, + ); + + /// Emits a subgraph.indexing.agreement.accepted event durably. + /// + /// Same durability contract as + /// [`emit_subgraph_indexing_agreement_terminated`](Self::emit_subgraph_indexing_agreement_terminated): + /// synchronous send, `Ok(())` lets the caller stamp its + /// `accepted_event_emitted_at` marker, `Err(_)` leaves it unset for the next + /// sweep. At-least-once; consumers dedup on `event.agreement_id`. + async fn emit_subgraph_indexing_agreement_accepted( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementAccepted, + ) -> Result<(), EmitError>; + + /// Emits a subgraph.indexing.agreement.request.expired event durably. + /// + /// Same durability contract as the other `emit_*` methods: synchronous send, + /// `Ok(())` lets the caller stamp its `expired_event_emitted_at` marker, + /// `Err(_)` leaves it unset for the next sweep. At-least-once; consumers dedup + /// on `event.agreement_id`. + async fn emit_subgraph_indexing_agreement_request_expired( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementRequestExpired, + ) -> Result<(), EmitError>; + + /// Produces a subgraph.indexing.agreement.n_indexers_unavailable event + fn produce_subgraph_indexing_agreement_n_indexers_unavailable( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementNIndexersUnavailable, + ); + + /// Emits a subgraph.indexing.agreement.terminated event durably. + /// + /// Sends synchronously and reports the outcome. `Ok(())` means the broker + /// accepted the event (or emission is disabled, a no-op the caller can treat + /// as success); the caller may then stamp its `terminated_event_emitted_at` + /// marker. `Err(_)` means the send failed and the marker must stay unset so + /// the next sweep re-derives and retries. Delivery is at-least-once; + /// consumers dedup on `event.agreement_id`. + async fn emit_subgraph_indexing_agreement_terminated( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementTerminated, + ) -> Result<(), EmitError>; + + /// Flush buffered diagnostic events on shutdown. Best-effort and bounded; the + /// default is a no-op for implementations with nothing to flush. + async fn flush(&self) {} +} + +/// A broker handle that may be absent while (re)connecting. +type SharedProducer = Arc>>>; + +/// Kafka producer wrapper for Subgraph Indexing agreements lifecycle events. +/// +/// `streaming == None` means emission is disabled by configuration -- durable +/// `emit_*` calls are a successful no-op so the caller stamps its marker, and +/// diagnostic `produce_*` calls are dropped. `streaming == Some` means emission +/// is on; the inner producer may still be `None` while the broker is being +/// (re)connected, in which case durable emits return `Err` (so markers are not +/// stamped) and diagnostic events are dropped. +pub struct SubgraphIndexingAgreementsEventsEmitter { + streaming: Option, +} + +/// State for an emitter with event streaming enabled. +struct Streaming { + /// Current broker handle. `None` while (re)connecting. + producer: SharedProducer, + /// Bounded channel for the best-effort diagnostic tier. + queue: mpsc::Sender, + /// Cumulative diagnostic events dropped (queue full, or the broker was + /// disconnected). Surfaced in the warn logs since there is no metrics stack. + dropped: Arc, + /// Signals the drain task to flush what is buffered and exit (used by `flush` + /// on shutdown). `notify_one` stores a permit, so the signal is not lost if + /// the drain task is mid-send when it fires. + shutdown: Arc, + /// The drain task handle, awaited by `flush`. + drain_handle: Mutex>>, +} + +impl SubgraphIndexingAgreementsEventsEmitter { + /// Creates a disabled producer (emission off by configuration). + pub fn disabled() -> Self { + Self { streaming: None } + } + + /// Creates a streaming producer. The broker is connected in the background + /// (retrying if unreachable at startup), so this never blocks and never comes + /// up permanently disabled on a transient broker outage. + pub fn enabled(config: KafkaConfig, capacity: usize) -> Self { + let producer: SharedProducer = Arc::new(Mutex::new(None)); + let dropped = Arc::new(AtomicU64::new(0)); + let shutdown = Arc::new(Notify::new()); + let (tx, mut rx) = mpsc::channel::(capacity); + + // Connect (retrying) and swap the handle in. rskafka reconnects internally + // once connected, so this task only bridges an unreachable-at-startup + // broker, then exits. + Self::spawn_connect_task(config, producer.clone()); + + // Drain diagnostic events through the current producer, dropping (with a + // counter) while disconnected. On shutdown, flush what is buffered. + let drain_handle = { + let producer = producer.clone(); + let dropped = dropped.clone(); + let shutdown = shutdown.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + maybe = rx.recv() => match maybe { + Some(event) => Self::drain_one(&producer, &dropped, event).await, + None => break, + }, + _ = shutdown.notified() => { + while let Ok(event) = rx.try_recv() { + Self::drain_one(&producer, &dropped, event).await; + } + break; + } + } + } + }) + }; + + Self { + streaming: Some(Streaming { + producer, + queue: tx, + dropped, + shutdown, + drain_handle: Mutex::new(Some(drain_handle)), + }), + } + } + + /// Retry `KafkaProducer::new` with capped backoff until it connects, then swap + /// the handle in and exit. Logs the first failure and then periodically so an + /// operator without dashboards still sees that emission is off. + fn spawn_connect_task(config: KafkaConfig, producer: SharedProducer) { + tokio::spawn(async move { + const MAX_BACKOFF_SECS: u64 = 30; + const WARN_EVERY: u32 = 5; + let mut attempt: u32 = 0; + loop { + match KafkaProducer::new(&config).await { + Ok(client) => { + // Recover from poisoning rather than panic: the guarded + // handle is always valid, so a prior panic elsewhere must + // not take down the connect task. + *producer.lock().unwrap_or_else(|e| e.into_inner()) = + Some(Arc::new(client)); + tracing::info!("Subgraph Indexing Agreement event broker connected"); + return; + } + Err(err) => { + attempt = attempt.saturating_add(1); + if attempt == 1 || attempt.is_multiple_of(WARN_EVERY) { + tracing::warn!( + attempt, + error = %err, + "event broker unreachable; lifecycle events are NOT being sent -- retrying" + ); + } + let backoff = MAX_BACKOFF_SECS.min(2u64.saturating_pow(attempt.min(5))); + tokio::time::sleep(Duration::from_secs(backoff)).await; + } + } + } + }); + } + + /// Read the current broker handle, recovering from a poisoned lock rather + /// than panicking (the guarded `Option` is always valid). + fn current_producer( + producer: &Mutex>>, + ) -> Option> { + producer.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + /// Send one diagnostic event through the current producer, or drop it (with a + /// counter) if the broker is not connected yet. + async fn drain_one( + producer: &Mutex>>, + dropped: &AtomicU64, + event: QueuedSubgraphIndexingAgreementEvent, + ) { + let Some(handle) = Self::current_producer(producer) else { + let total = dropped.fetch_add(1, Ordering::Relaxed) + 1; + tracing::warn!( + dropped_total = total, + "event broker not connected; dropping diagnostic event" + ); + return; + }; + let (event_type, key, envelope) = Self::prepare_event(event); + // `send_event` already bounds the produce attempt via `KafkaProducer`'s + // internal produce timeout, so no additional timeout is layered here. + Self::send_event(&handle, event_type, &key, envelope).await; + } +} + +#[async_trait] +impl SubgraphIndexingAgreementEventsProducer for SubgraphIndexingAgreementsEventsEmitter { + fn produce_subgraph_indexing_agreement_request_received( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementRequestReceived, + ) { + self.enqueue( + subgraph_deployment_qm_hash, + the_graph_network.into(), + EventPayload::RequestReceived(event), + ); + } + + fn produce_subgraph_indexing_agreement_proposed( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementProposed, + ) { + self.enqueue( + subgraph_deployment_qm_hash, + the_graph_network.into(), + EventPayload::Proposed(event), + ); + } + + async fn emit_subgraph_indexing_agreement_accepted( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementAccepted, + ) -> Result<(), EmitError> { + self.send_durable( + subgraph_deployment_qm_hash, + the_graph_network.into(), + SubgraphIndexingAgreementEventType::Accepted, + EventPayload::Accepted(event), + ) + .await + } + + async fn emit_subgraph_indexing_agreement_request_expired( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementRequestExpired, + ) -> Result<(), EmitError> { + self.send_durable( + subgraph_deployment_qm_hash, + the_graph_network.into(), + SubgraphIndexingAgreementEventType::RequestExpired, + EventPayload::RequestExpired(event), + ) + .await + } + + fn produce_subgraph_indexing_agreement_n_indexers_unavailable( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementNIndexersUnavailable, + ) { + self.enqueue( + subgraph_deployment_qm_hash, + the_graph_network.into(), + EventPayload::NIndexersUnavailable(event), + ); + } + + async fn emit_subgraph_indexing_agreement_terminated( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: ChainId, + event: proto::SubgraphIndexingAgreementTerminated, + ) -> Result<(), EmitError> { + self.send_durable( + subgraph_deployment_qm_hash, + the_graph_network.into(), + SubgraphIndexingAgreementEventType::Terminated, + EventPayload::Terminated(event), + ) + .await + } + + async fn flush(&self) { + self.flush_diagnostics().await; + } +} + +impl SubgraphIndexingAgreementsEventsEmitter { + /// Send a durable-tier event synchronously and report the outcome. Shared by + /// the `emit_*` methods. + /// + /// - Disabled by config: a successful no-op so the caller stamps its marker. + /// - Streaming but not yet connected: `Err`, so the caller does NOT stamp its + /// marker and the event is retried on the next sweep once the broker is up. + /// - Connected: send synchronously and report the broker's outcome. + async fn send_durable( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: Caip2ChainId, + event_type: SubgraphIndexingAgreementEventType, + payload: EventPayload, + ) -> Result<(), EmitError> { + let Some(streaming) = &self.streaming else { + return Ok(()); + }; + let Some(producer) = Self::current_producer(&streaming.producer) else { + return Err(EmitError::Send( + "event broker not connected; will retry".to_string(), + )); + }; + + let metadata = EventMetadata { + subgraph_deployment_qm_hash, + the_graph_network, + }; + let key = metadata.partition_key(); + let envelope = Self::create_event_envelope(event_type, &metadata, payload.into_proto()); + + let mut buf = Vec::with_capacity(envelope.encoded_len()); + envelope + .encode(&mut buf) + .map_err(|e| EmitError::Send(format!("encode failed: {e}")))?; + + producer + .send(&key, &buf) + .await + .map_err(|e| EmitError::Send(e.to_string()))?; + Ok(()) + } + + fn enqueue( + &self, + subgraph_deployment_qm_hash: DeploymentId, + the_graph_network: Caip2ChainId, + payload: EventPayload, + ) { + let Some(streaming) = &self.streaming else { + return; + }; + + let event = QueuedSubgraphIndexingAgreementEvent { + metadata: EventMetadata { + subgraph_deployment_qm_hash, + the_graph_network, + }, + payload, + }; + + let event_type = event.payload.event_type(); + if let Err(err) = streaming.queue.try_send(event) { + let total = streaming.dropped.fetch_add(1, Ordering::Relaxed) + 1; + let reason = match err { + mpsc::error::TrySendError::Full(_) => "event queue full", + mpsc::error::TrySendError::Closed(_) => "event queue closed", + }; + tracing::warn!( + event_type = %event_type, + dropped_total = total, + "{reason}; dropping diagnostic event" + ); + } + } + + /// Flush buffered diagnostic events and stop the drain task. Called once on + /// shutdown. Bounded by a timeout so a slow/unreachable broker can't hang + /// shutdown. Idempotent: the drain handle is taken, so a second call is a + /// no-op. + async fn flush_diagnostics(&self) { + let Some(streaming) = &self.streaming else { + return; + }; + // Wake the drain task (permit is stored if it is mid-send). + streaming.shutdown.notify_one(); + let handle = streaming + .drain_handle + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take(); + if let Some(handle) = handle { + const FLUSH_TIMEOUT: Duration = Duration::from_secs(5); + if tokio::time::timeout(FLUSH_TIMEOUT, handle).await.is_err() { + tracing::warn!("timed out flushing buffered lifecycle events on shutdown"); + } + } + } + + fn prepare_event( + event: QueuedSubgraphIndexingAgreementEvent, + ) -> ( + SubgraphIndexingAgreementEventType, + String, + proto::SubgraphIndexingAgreementEvent, + ) { + let event_type = event.payload.event_type(); + let key = event.metadata.partition_key(); + let envelope = + Self::create_event_envelope(event_type, &event.metadata, event.payload.into_proto()); + + (event_type, key, envelope) + } + + /// Sends the event to the Kafka topic + /// + /// Errors are logged, but do not fail as it is a best-effort + async fn send_event( + producer: &Arc, + event_type: SubgraphIndexingAgreementEventType, + key: &str, + event: proto::SubgraphIndexingAgreementEvent, + ) { + let mut buf = Vec::with_capacity(event.encoded_len()); + if let Err(e) = event.encode(&mut buf) { + tracing::error!( + event_type = %event_type, + error = %e, + "failed to encode Subgraph Indexing Agreement event" + ); + return; + } + + if let Err(e) = producer.send(key, &buf).await { + tracing::warn!( + event_type = %event_type, + key, + error = %e, + "failed to send Subgraph Indexing Agreement event to producer (event dropped)" + ); + } + } + + /// Creates the event envelope with common metadata + fn create_event_envelope( + event_type: SubgraphIndexingAgreementEventType, + metadata: &EventMetadata, + payload: proto::subgraph_indexing_agreement_event::Payload, + ) -> proto::SubgraphIndexingAgreementEvent { + proto::SubgraphIndexingAgreementEvent { + event_id: uuid::Uuid::now_v7().to_string(), + event_type: event_type.to_string(), + event_version: "1.0".to_string(), + timestamp: chrono::Utc::now().to_rfc3339(), + subgraph_deployment_qm_hash: metadata.subgraph_deployment_qm_hash.to_string(), + the_graph_network: metadata.the_graph_network.to_string(), + payload: Some(payload), + } + } +} + +/// A queued event: shared routing metadata plus its type-specific payload. +struct QueuedSubgraphIndexingAgreementEvent { + metadata: EventMetadata, + payload: EventPayload, +} + +/// Routing metadata common to every Subgraph Indexing Agreement event. +struct EventMetadata { + /// The Subgraph deployment. Rendered to its `Qm...` hash representation + /// when building the envelope and partition key. + subgraph_deployment_qm_hash: DeploymentId, + /// The Graph protocol network. Rendered to its CAIP-2 form + /// (e.g. `eip155:42161`) when building the envelope and partition key. + the_graph_network: Caip2ChainId, +} + +impl EventMetadata { + /// Creates the partition key for the Subgraph Indexing Agreement events + /// + /// Format: `{the_graph_network}/{subgraph_deployment_qm_hash}` + /// e.g. `eip155:42161/QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9` + fn partition_key(&self) -> String { + format!( + "{}/{}", + self.the_graph_network, self.subgraph_deployment_qm_hash + ) + } +} + +/// The type-specific payload of a Subgraph Indexing Agreement event. +enum EventPayload { + RequestReceived(proto::SubgraphIndexingAgreementRequestReceived), + Proposed(proto::SubgraphIndexingAgreementProposed), + Accepted(proto::SubgraphIndexingAgreementAccepted), + RequestExpired(proto::SubgraphIndexingAgreementRequestExpired), + NIndexersUnavailable(proto::SubgraphIndexingAgreementNIndexersUnavailable), + Terminated(proto::SubgraphIndexingAgreementTerminated), +} + +impl EventPayload { + fn event_type(&self) -> SubgraphIndexingAgreementEventType { + match self { + Self::RequestReceived(_) => SubgraphIndexingAgreementEventType::RequestReceived, + Self::Proposed(_) => SubgraphIndexingAgreementEventType::Proposed, + Self::Accepted(_) => SubgraphIndexingAgreementEventType::Accepted, + Self::RequestExpired(_) => SubgraphIndexingAgreementEventType::RequestExpired, + Self::NIndexersUnavailable(_) => { + SubgraphIndexingAgreementEventType::NIndexersUnavailable + } + Self::Terminated(_) => SubgraphIndexingAgreementEventType::Terminated, + } + } + + fn into_proto(self) -> proto::subgraph_indexing_agreement_event::Payload { + use proto::subgraph_indexing_agreement_event::Payload; + match self { + Self::RequestReceived(e) => Payload::SubgraphIndexingAgreementRequestReceived(e), + Self::Proposed(e) => Payload::SubgraphIndexingAgreementProposed(e), + Self::Accepted(e) => Payload::SubgraphIndexingAgreementAccepted(e), + Self::RequestExpired(e) => Payload::SubgraphIndexingAgreementRequestExpired(e), + Self::NIndexersUnavailable(e) => { + Payload::SubgraphIndexingAgreementNIndexersUnavailable(e) + } + Self::Terminated(e) => Payload::SubgraphIndexingAgreementTerminated(e), + } + } +} + +/// Subgraph Indexing Agreement Event type discriminator for events +#[derive(Debug, Clone, Copy)] +enum SubgraphIndexingAgreementEventType { + RequestReceived, + Proposed, + Accepted, + RequestExpired, + NIndexersUnavailable, + Terminated, +} + +impl std::fmt::Display for SubgraphIndexingAgreementEventType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + Self::RequestReceived => "subgraph.indexing.agreement.request.received", + Self::Proposed => "subgraph.indexing.agreement.proposed", + Self::Accepted => "subgraph.indexing.agreement.accepted", + Self::RequestExpired => "subgraph.indexing.agreement.request.expired", + Self::NIndexersUnavailable => "subgraph.indexing.agreement.n_indexers_unavailable", + Self::Terminated => "subgraph.indexing.agreement.terminated", + }; + f.write_str(s) + } +} + +#[cfg(test)] +mod tests { + use proto::subgraph_indexing_agreement_event::Payload; + + use super::*; + + const HASH: &str = "QmTXzATwNfgGVukV1fX2T6xw9f6LAYRVWpsdXyRWzUR2H9"; + const CHAIN_ID: ChainId = 42161; + const NETWORK: &str = "eip155:42161"; + + fn deployment_id() -> DeploymentId { + HASH.parse().expect("HASH is a valid deployment id") + } + + /// A config whose producer can never connect (`partitions == 0` fails + /// `KafkaProducer::new` fast, before any network), so the emitter stays in the + /// disconnected state for the test. + fn never_connects_config() -> KafkaConfig { + KafkaConfig { + brokers: vec![], + topic: "test".to_string(), + partitions: 0, + sasl_mechanism: None, + sasl_username: None, + sasl_password: None, + tls_enabled: false, + tls_ca_cert_path: None, + } + } + + #[tokio::test] + async fn disabled_emitter_durable_emit_is_ok_noop() { + // Disabled by config: durable emit is a successful no-op, so the caller + // stamps its marker (events are intentionally off). Diagnostic produce is + // a no-op too (must not panic). + let emitter = SubgraphIndexingAgreementsEventsEmitter::disabled(); + + let res = emitter + .emit_subgraph_indexing_agreement_terminated( + deployment_id(), + CHAIN_ID, + proto::SubgraphIndexingAgreementTerminated::default(), + ) + .await; + assert!(res.is_ok(), "disabled durable emit must be Ok (no-op)"); + + emitter.produce_subgraph_indexing_agreement_n_indexers_unavailable( + deployment_id(), + CHAIN_ID, + proto::SubgraphIndexingAgreementNIndexersUnavailable::default(), + ); + emitter.flush().await; + } + + #[tokio::test] + async fn streaming_but_disconnected_durable_emit_errors() { + // Enabled but the broker never connects: durable emit must return Err so + // the caller does NOT stamp its marker and the event is retried on the next + // sweep. This is what prevents a broker-down-at-startup from silently + // losing durable events. + let emitter = SubgraphIndexingAgreementsEventsEmitter::enabled(never_connects_config(), 8); + + let res = emitter + .emit_subgraph_indexing_agreement_terminated( + deployment_id(), + CHAIN_ID, + proto::SubgraphIndexingAgreementTerminated::default(), + ) + .await; + assert!( + res.is_err(), + "disconnected durable emit must be Err so the sweep retries" + ); + + // Diagnostic produce + flush must not panic while disconnected. + emitter.produce_subgraph_indexing_agreement_n_indexers_unavailable( + deployment_id(), + CHAIN_ID, + proto::SubgraphIndexingAgreementNIndexersUnavailable::default(), + ); + emitter.flush().await; + } + + fn metadata() -> EventMetadata { + EventMetadata { + subgraph_deployment_qm_hash: deployment_id(), + the_graph_network: CHAIN_ID.into(), + } + } + + /// Every payload variant paired with the event-type string and proto payload + /// variant it is contractually required to map to. Adding a new event variant + /// without updating this table is a compile error (non-exhaustive match below). + fn all_payloads() -> Vec<(EventPayload, &'static str)> { + vec![ + ( + EventPayload::RequestReceived(Default::default()), + "subgraph.indexing.agreement.request.received", + ), + ( + EventPayload::Proposed(Default::default()), + "subgraph.indexing.agreement.proposed", + ), + ( + EventPayload::Accepted(Default::default()), + "subgraph.indexing.agreement.accepted", + ), + ( + EventPayload::RequestExpired(Default::default()), + "subgraph.indexing.agreement.request.expired", + ), + ( + EventPayload::NIndexersUnavailable(Default::default()), + "subgraph.indexing.agreement.n_indexers_unavailable", + ), + ( + EventPayload::Terminated(Default::default()), + "subgraph.indexing.agreement.terminated", + ), + ] + } + + #[test] + fn caip2_chain_id_from_chain_id_renders_eip155() { + let network: Caip2ChainId = CHAIN_ID.into(); + assert_eq!(network.to_string(), NETWORK); + assert_eq!(network.to_string(), "eip155:42161"); + } + + #[test] + fn partition_key_uses_network_slash_hash_format() { + assert_eq!(metadata().partition_key(), format!("{NETWORK}/{HASH}")); + } + + #[test] + fn event_type_string_matches_payload_for_every_variant() { + for (payload, expected) in all_payloads() { + assert_eq!(payload.event_type().to_string(), expected); + } + } + + #[test] + fn into_proto_preserves_the_variant_for_every_payload() { + for (payload, expected) in all_payloads() { + // Guards against a copy-paste swap between two payload arms. + let mapped = match payload.into_proto() { + Payload::SubgraphIndexingAgreementRequestReceived(_) => { + "subgraph.indexing.agreement.request.received" + } + Payload::SubgraphIndexingAgreementProposed(_) => { + "subgraph.indexing.agreement.proposed" + } + Payload::SubgraphIndexingAgreementAccepted(_) => { + "subgraph.indexing.agreement.accepted" + } + Payload::SubgraphIndexingAgreementRequestExpired(_) => { + "subgraph.indexing.agreement.request.expired" + } + Payload::SubgraphIndexingAgreementNIndexersUnavailable(_) => { + "subgraph.indexing.agreement.n_indexers_unavailable" + } + Payload::SubgraphIndexingAgreementTerminated(_) => { + "subgraph.indexing.agreement.terminated" + } + }; + assert_eq!(mapped, expected); + } + } + + #[test] + fn create_event_envelope_populates_common_metadata() { + let payload = + EventPayload::RequestReceived(proto::SubgraphIndexingAgreementRequestReceived { + agreements_requested: 2, + }); + let event_type = payload.event_type(); + let envelope = SubgraphIndexingAgreementsEventsEmitter::create_event_envelope( + event_type, + &metadata(), + payload.into_proto(), + ); + + assert_eq!( + envelope.event_type, + "subgraph.indexing.agreement.request.received" + ); + assert_eq!(envelope.event_version, "1.0"); + assert_eq!(envelope.subgraph_deployment_qm_hash, HASH); + assert_eq!(envelope.the_graph_network, NETWORK); + assert!(matches!( + envelope.payload, + Some(Payload::SubgraphIndexingAgreementRequestReceived(_)) + )); + + let id = uuid::Uuid::parse_str(&envelope.event_id).expect("event_id is a valid uuid"); + assert_eq!(id.get_version_num(), 7, "event_id should be a UUIDv7"); + chrono::DateTime::parse_from_rfc3339(&envelope.timestamp) + .expect("timestamp is valid rfc3339"); + } + + #[test] + fn deployment_id_renders_to_qm_hash_in_envelope_and_partition_key() { + // The emitter takes a strongly-typed DeploymentId and is responsible for + // rendering it to its Qm-hash string on the wire. Assert that rendering is + // the canonical Qm hash (round-trips back to the same DeploymentId) and that + // the partition key uses the same rendering. + let dep = deployment_id(); + let envelope = SubgraphIndexingAgreementsEventsEmitter::create_event_envelope( + SubgraphIndexingAgreementEventType::RequestReceived, + &metadata(), + EventPayload::RequestReceived(Default::default()).into_proto(), + ); + + assert_eq!(envelope.subgraph_deployment_qm_hash, HASH); + let parsed: DeploymentId = envelope + .subgraph_deployment_qm_hash + .parse() + .expect("envelope qm hash parses back into a DeploymentId"); + assert_eq!(parsed, dep, "qm-hash rendering must be lossless"); + + assert_eq!( + metadata().partition_key(), + format!("eip155:42161/{dep}"), + "partition key must use the caip2 network and the Qm-hash rendering" + ); + } + + #[test] + fn prepare_event_builds_key_type_and_wire_encodable_envelope() { + let event = QueuedSubgraphIndexingAgreementEvent { + metadata: metadata(), + payload: EventPayload::Terminated(proto::SubgraphIndexingAgreementTerminated { + indexer: "0xabc".to_string(), + terminated_at: 42, + terminated_by: "0xdef".to_string(), + terminated_tx: "0x123".to_string(), + ..Default::default() + }), + }; + + let (event_type, key, envelope) = + SubgraphIndexingAgreementsEventsEmitter::prepare_event(event); + + assert_eq!( + event_type.to_string(), + "subgraph.indexing.agreement.terminated" + ); + assert_eq!(key, format!("{NETWORK}/{HASH}")); + + // Round-trips through the same prost encode path `send_event` uses. + let mut buf = Vec::with_capacity(envelope.encoded_len()); + envelope.encode(&mut buf).expect("envelope encodes"); + let decoded = + proto::SubgraphIndexingAgreementEvent::decode(&buf[..]).expect("envelope decodes"); + assert_eq!(decoded, envelope); + } +} diff --git a/dipper-producer/src/kafka.rs b/dipper-producer/src/kafka.rs new file mode 100644 index 00000000..25714154 --- /dev/null +++ b/dipper-producer/src/kafka.rs @@ -0,0 +1,27 @@ +//! Kafka client for subgraph indexing agreement event streaming. +//! +//! This module provides a Kafka producer for emitting subgraph indexing agreement lifecycle events +//! to a kafka topic +//! +//! Events are encoded using Protocol Buffers for compact, schema-enforced messages. +//! +//! # Example +//! +//! ```ignore +//! use dipper_producer::kafka::{KafkaConfig, KafkaProducer}; +//! +//! let config = KafkaConfig { +//! brokers: vec!["localhost:9092".to_string()], +//! topic: "dipper.subgraph.indexing.agreement.events".to_string(), +//! partitions: 16, +//! }; +//! +//! let producer = KafkaProducer::new(&config).await?; +//! +//! // Send an event with partition key and protobuf payload +//! producer.send("QmT329Bej8AwSLahmgnmi6fdYkj3rorYAcCes45gDv9aJ4", &encoded_event).await?; +//! ``` + +mod producer; + +pub use producer::{Error, KafkaConfig, KafkaProducer}; diff --git a/dipper-producer/src/kafka/producer.rs b/dipper-producer/src/kafka/producer.rs new file mode 100644 index 00000000..ed99a38f --- /dev/null +++ b/dipper-producer/src/kafka/producer.rs @@ -0,0 +1,390 @@ +//! Kafka producer for sending dipper events on a configured topic + +use std::{ + path::{Path, PathBuf}, + sync::{Arc, Once}, + time::Duration, +}; + +use rskafka::{ + client::{ + ClientBuilder, Credentials, SaslConfig, + partition::{Compression, PartitionClient, UnknownTopicHandling}, + }, + record::Record, +}; +use rustls::ClientConfig; + +static RUSTLS_CRYPTO_PROVIDER: Once = Once::new(); + +/// Kafka producer configuration. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct KafkaConfig { + /// Kafka broker addresses. + pub brokers: Vec, + /// Kafka topic name. + #[serde(default = "default_kafka_topic")] + pub topic: String, + /// Number of partitions used for key-based partition hashing. + #[serde(default = "default_kafka_partitions")] + pub partitions: u32, + /// SASL authentication mechanism (e.g., "PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"). + #[serde(default)] + pub sasl_mechanism: Option, + /// SASL username. + #[serde(default)] + pub sasl_username: Option, + /// SASL password. + #[serde(default)] + pub sasl_password: Option, + /// Enable TLS encryption. + #[serde(default)] + pub tls_enabled: bool, + /// Path to a PEM-encoded CA certificate file for TLS verification. + #[serde(default)] + pub tls_ca_cert_path: Option, +} + +/// Default Kafka topic for subgraph indexing agreement events. +pub fn default_kafka_topic() -> String { + "dipper.subgraph.indexing.agreement.events".to_string() +} + +/// Default number of Kafka partitions used for key-based partition hashing. +pub fn default_kafka_partitions() -> u32 { + 16 +} + +/// Kafka producer for sending worker events. +/// +/// The producer is thread-safe and can be shared across tasks via `Arc`. +pub struct KafkaProducer { + partitions: u32, + partition_clients: Vec>, +} + +impl KafkaProducer { + const PRODUCE_TIMEOUT: Duration = Duration::from_secs(30); + + /// Creates a new Kafka producer with the given configuration. + pub async fn new(config: &KafkaConfig) -> Result { + if config.partitions == 0 { + return Err(Error::InvalidPartitionCount); + } + + let mut builder = ClientBuilder::new(config.brokers.clone()); + + // Configure SASL authentication if mechanism is specified + if let Some(mechanism_str) = &config.sasl_mechanism { + let mechanism: SaslMechanism = mechanism_str.parse()?; + let sasl_config = Self::build_sasl_config(mechanism, config)?; + builder = builder.sasl_config(sasl_config); + } + + // Configure TLS if enabled + if config.tls_enabled { + let tls_config = Self::build_tls_config(config.tls_ca_cert_path.as_deref())?; + builder = builder.tls_config(tls_config); + } + + let client = builder.build().await.map_err(Error::Connection)?; + let client = Arc::new(client); + let mut partition_clients = Vec::with_capacity(config.partitions as usize); + + for partition in 0..config.partitions { + let partition_client = Arc::new( + client + .partition_client(&config.topic, partition as i32, UnknownTopicHandling::Error) + .await + .map_err(Error::PartitionClient)?, + ); + partition_clients.push(partition_client); + } + + Ok(Self { + partitions: config.partitions, + partition_clients, + }) + } + + /// Builds SASL configuration from the provided mechanism and credentials. + fn build_sasl_config( + mechanism: SaslMechanism, + config: &KafkaConfig, + ) -> Result { + let username = config + .sasl_username + .clone() + .ok_or(Error::MissingSaslUsername)?; + let password = config + .sasl_password + .clone() + .ok_or(Error::MissingSaslPassword)?; + + let credentials = Credentials::new(username, password); + + Ok(match mechanism { + SaslMechanism::Plain => SaslConfig::Plain(credentials), + SaslMechanism::ScramSha256 => SaslConfig::ScramSha256(credentials), + SaslMechanism::ScramSha512 => SaslConfig::ScramSha512(credentials), + }) + } + + /// Builds TLS configuration. + /// + /// If a custom CA certificate path is provided, the client will trust that CA + /// for verifying broker connections. Otherwise, system root certificates are used. + fn build_tls_config(ca_cert_path: Option<&Path>) -> Result, Error> { + install_rustls_crypto_provider(); + + let root_store = match ca_cert_path { + Some(path) => { + let ca_pem = fs_err::read(path).map_err(|e| Error::TlsCaCert { source: e })?; + let mut reader = std::io::BufReader::new(&ca_pem[..]); + let certs: Vec<_> = rustls_pemfile::certs(&mut reader) + .collect::>() + .map_err(|e| Error::TlsCaCert { source: e })?; + + let mut store = rustls::RootCertStore::empty(); + for cert in certs { + store.add(cert).map_err(|e| Error::TlsCaCert { + source: std::io::Error::new(std::io::ErrorKind::InvalidData, e), + })?; + } + store + } + None => rustls::RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(), + }, + }; + + let tls_config = ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + Ok(Arc::new(tls_config)) + } + + /// Sends an event to Kafka. + /// + /// Events are partitioned by the partition key (table discriminator) before being written to + /// Kafka. The produce attempt times out after 30 seconds. + pub async fn send(&self, partition_key: &str, payload: &[u8]) -> Result<(), Error> { + let partition = self.partition_for_key(partition_key); + let partition_client = &self.partition_clients[partition as usize]; + + let record = Record { + key: Some(partition_key.as_bytes().to_vec()), + value: Some(payload.to_vec()), + headers: Default::default(), + timestamp: chrono::Utc::now(), + }; + + tokio::time::timeout( + Self::PRODUCE_TIMEOUT, + partition_client.produce(vec![record], Compression::Gzip), + ) + .await + .map_err(|_| Error::Timeout)? + .map_err(Error::Send) + .map(|_| ()) + } + + /// Computes the partition for a given key. + /// + /// Uses a deterministic FNV-1a hash modulo the partition count so that a given + /// key always maps to the same partition across restarts and instances, + /// preserving per-key ordering. The partition count is configured via + /// `KafkaConfig::partitions`. + fn partition_for_key(&self, key: &str) -> i32 { + // FNV-1a (32-bit): order-dependent and well-distributed, unlike a byte sum. + const FNV_OFFSET_BASIS: u32 = 0x811c_9dc5; + const FNV_PRIME: u32 = 0x0100_0193; + let hash = key.bytes().fold(FNV_OFFSET_BASIS, |hash, b| { + (hash ^ u32::from(b)).wrapping_mul(FNV_PRIME) + }); + // `partitions` is guaranteed non-zero by `KafkaProducer::new`. + (hash % self.partitions) as i32 + } +} + +fn install_rustls_crypto_provider() { + RUSTLS_CRYPTO_PROVIDER.call_once(|| { + // Necessary for the Kafka client: it builds a Rustls TLS config directly, + // so install a provider before `ClientConfig::builder()` tries to infer one. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + }); +} + +/// Errors that can occur when working with the Kafka producer. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Failed to connect to Kafka brokers + #[error("failed to connect to Kafka brokers")] + Connection(#[source] rskafka::client::error::Error), + + /// Failed to get partition client + #[error("failed to get partition client")] + PartitionClient(#[source] rskafka::client::error::Error), + + /// Failed to send event to Kafka + #[error("failed to send event to Kafka")] + Send(#[source] rskafka::client::error::Error), + + /// Kafka operation timed out + #[error("Kafka operation timed out")] + Timeout, + + /// Partition count must be greater than zero + #[error("partitions must be greater than zero")] + InvalidPartitionCount, + + /// Unsupported SASL mechanism + #[error("unsupported SASL mechanism '{0}', supported: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512")] + UnsupportedSaslMechanism(String), + + /// Missing SASL username + #[error("sasl_username is required when sasl_mechanism is set")] + MissingSaslUsername, + + /// Missing SASL password + #[error("sasl_password is required when sasl_mechanism is set")] + MissingSaslPassword, + + /// Failed to load TLS CA certificate + #[error("failed to load TLS CA certificate")] + TlsCaCert { + #[source] + source: std::io::Error, + }, +} + +/// Supported SASL authentication mechanisms. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SaslMechanism { + Plain, + ScramSha256, + ScramSha512, +} + +impl std::str::FromStr for SaslMechanism { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s.to_uppercase().as_str() { + "PLAIN" => Ok(Self::Plain), + "SCRAM-SHA-256" => Ok(Self::ScramSha256), + "SCRAM-SHA-512" => Ok(Self::ScramSha512), + _ => Err(Error::UnsupportedSaslMechanism(s.to_string())), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn should_parse_sasl_mechanism_correctly() { + // Case-insensitive parsing + assert_eq!( + "PLAIN".parse::().unwrap(), + SaslMechanism::Plain + ); + assert_eq!( + "plain".parse::().unwrap(), + SaslMechanism::Plain + ); + assert_eq!( + "SCRAM-SHA-256".parse::().unwrap(), + SaslMechanism::ScramSha256 + ); + assert_eq!( + "scram-sha-256".parse::().unwrap(), + SaslMechanism::ScramSha256 + ); + assert_eq!( + "SCRAM-SHA-512".parse::().unwrap(), + SaslMechanism::ScramSha512 + ); + + // Unsupported mechanism + assert!("GSSAPI".parse::().is_err()); + } + + #[test] + fn should_build_sasl_plain_sasl_config() { + let config = make_kafka_config(Some("PLAIN"), Some("user".into()), Some("pass".into())); + let result = KafkaProducer::build_sasl_config(SaslMechanism::Plain, &config); + assert!(matches!(result, Ok(SaslConfig::Plain(_)))); + } + + #[test] + fn should_build_scram_sha_256_sasl_config() { + let config = make_kafka_config( + Some("SCRAM-SHA-256"), + Some("user".into()), + Some("pass".into()), + ); + let result = KafkaProducer::build_sasl_config(SaslMechanism::ScramSha256, &config); + assert!(matches!(result, Ok(SaslConfig::ScramSha256(_)))); + } + + #[test] + fn should_build_sasl_sha_512_sasl_config() { + let config = make_kafka_config( + Some("SCRAM-SHA-512"), + Some("user".into()), + Some("pass".into()), + ); + let result = KafkaProducer::build_sasl_config(SaslMechanism::ScramSha512, &config); + assert!(matches!(result, Ok(SaslConfig::ScramSha512(_)))); + } + + #[test] + fn should_throw_err_on_missing_credentials() { + // Missing username + let config = make_kafka_config(Some("PLAIN"), None, Some("pass".into())); + assert!(matches!( + KafkaProducer::build_sasl_config(SaslMechanism::Plain, &config), + Err(Error::MissingSaslUsername) + )); + + // Missing password + let config = make_kafka_config(Some("PLAIN"), Some("user".into()), None); + assert!(matches!( + KafkaProducer::build_sasl_config(SaslMechanism::Plain, &config), + Err(Error::MissingSaslPassword) + )); + } + + #[tokio::test] + async fn should_reject_zero_partitions() { + let mut config = make_kafka_config(None, None, None); + config.partitions = 0; + assert!(matches!( + KafkaProducer::new(&config).await, + Err(Error::InvalidPartitionCount) + )); + } + + // -------- Test helpers -------- + + /// Creates a test Kafka config with optional SASL credentials. + fn make_kafka_config( + mechanism: Option<&str>, + sasl_user: Option, + sasl_pass: Option, + ) -> KafkaConfig { + KafkaConfig { + brokers: vec!["localhost:9092".to_string()], + topic: "test".to_string(), + partitions: 1, + sasl_mechanism: mechanism.map(String::from), + sasl_username: sasl_user, + sasl_password: sasl_pass, + tls_enabled: false, + tls_ca_cert_path: None, + } + } +} diff --git a/dipper-producer/src/lib.rs b/dipper-producer/src/lib.rs new file mode 100644 index 00000000..71079c9c --- /dev/null +++ b/dipper-producer/src/lib.rs @@ -0,0 +1,3 @@ +pub mod events; +pub mod kafka; +pub mod proto; diff --git a/dipper-producer/src/proto/dipper.subgraph.indexing.agreement.events.v1.rs b/dipper-producer/src/proto/dipper.subgraph.indexing.agreement.events.v1.rs new file mode 100644 index 00000000..be209e64 --- /dev/null +++ b/dipper-producer/src/proto/dipper.subgraph.indexing.agreement.events.v1.rs @@ -0,0 +1,184 @@ +// This file is @generated by prost-build. +/// SubgraphIndexingAgreementEvent is the envelope that wraps all subgraph indexing agreement event types. +/// +/// Provides: +/// - event_id -> unique identifier (uuid) +/// - event_type -> event type discrimination +/// - event_version -> versioning for schema evolution awareness to the consumer +/// - timestamp -> when the event occurred, useful for ordering/debugging +/// - subgraph_deployment_qm_hash -> Qm hash of the Subgraph deployment +/// - the_graph_network -> CAIP-2 chain id of The Graph protocol network the Subgraph Deployment is published to (e.g. eip155:42161) +/// - payload -> the event payload, determined by the event_type +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubgraphIndexingAgreementEvent { + /// Unique event identifier. + /// Format: UUID v7 (time-ordered) for natural chronological sorting. + /// Example: "01912345-6789-7abc-def0-123456789abc" + #[prost(string, tag = "1")] + pub event_id: ::prost::alloc::string::String, + /// Event type discriminator for routing and filtering. + /// Values: "subgraph.indexing.agreement.request.received", "subgraph.indexing.agreement.proposed", "subgraph.indexing.agreement.accepted", "subgraph.indexing.agreement.request.expired", "subgraph.indexing.agreement.n_indexers_unavailable", "subgraph.indexing.agreement.terminated" + #[prost(string, tag = "2")] + pub event_type: ::prost::alloc::string::String, + /// Schema version for forward compatibility. + /// Consumers should handle unknown fields gracefully. + /// Current version: "1.0" + #[prost(string, tag = "3")] + pub event_version: ::prost::alloc::string::String, + /// Event timestamp in RFC 3339 format. + /// Example: "2024-01-15T10:30:00.123Z" + #[prost(string, tag = "4")] + pub timestamp: ::prost::alloc::string::String, + /// Qm hash of the Subgraph deployment with a submitted indexing agreement event. + #[prost(string, tag = "5")] + pub subgraph_deployment_qm_hash: ::prost::alloc::string::String, + /// CAIP-2 chain id of The Graph protocol network where the Subgraph was published to. + /// Format: "eip155:{chain_id}". Examples: "eip155:42161" (arbitrum), "eip155:421614" (arbitrum-sepolia) + #[prost(string, tag = "6")] + pub the_graph_network: ::prost::alloc::string::String, + /// Event payload - exactly one of the specific event types. + /// Use event_type field to determine which payload is present. + #[prost( + oneof = "subgraph_indexing_agreement_event::Payload", + tags = "7, 8, 9, 10, 11, 12" + )] + pub payload: ::core::option::Option, +} +/// Nested message and enum types in `SubgraphIndexingAgreementEvent`. +pub mod subgraph_indexing_agreement_event { + /// Event payload - exactly one of the specific event types. + /// Use event_type field to determine which payload is present. + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Payload { + #[prost(message, tag = "7")] + SubgraphIndexingAgreementRequestReceived( + super::SubgraphIndexingAgreementRequestReceived, + ), + #[prost(message, tag = "8")] + SubgraphIndexingAgreementProposed(super::SubgraphIndexingAgreementProposed), + #[prost(message, tag = "9")] + SubgraphIndexingAgreementAccepted(super::SubgraphIndexingAgreementAccepted), + #[prost(message, tag = "10")] + SubgraphIndexingAgreementRequestExpired( + super::SubgraphIndexingAgreementRequestExpired, + ), + #[prost(message, tag = "11")] + SubgraphIndexingAgreementNIndexersUnavailable( + super::SubgraphIndexingAgreementNIndexersUnavailable, + ), + #[prost(message, tag = "12")] + SubgraphIndexingAgreementTerminated(super::SubgraphIndexingAgreementTerminated), + } +} +/// SubgraphIndexingAgreementRequestReceived is emitted when a Subgraph developer submits a request to the dipper to find +/// indexers to index the given Subgraph. +/// +/// Event type: subgraph.indexing.agreement.request.received +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubgraphIndexingAgreementRequestReceived { + /// Number of indexing agreements requested by the Subgraph developer. + #[prost(int32, tag = "1")] + pub agreements_requested: i32, +} +/// SubgraphIndexingAgreementProposed is emitted once the dipper has gone to the IISA, received candidates (indexers), +/// sent gRPC offers to these candidates, but before any candidate has accepted onchain +/// +/// Event type: subgraph.indexing.agreement.proposed +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubgraphIndexingAgreementProposed { + /// 0x addresses of the Indexer candidates dipper sent agreement offers to in + /// this proposal round (the IISA-selected indexers newly proposed this cycle). + #[prost(string, repeated, tag = "1")] + pub candidates: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Timestamp of when the request expires + #[prost(uint64, tag = "2")] + pub request_expires_at: u64, +} +/// SubgraphIndexingAgreementAccepted is emitted when an Indexer candidate has accepted onchain the Subgraph indexing request. +/// +/// Event type: subgraph.indexing.agreement.accepted +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubgraphIndexingAgreementAccepted { + /// 0x hex of the on-chain agreement id (bytes16) that was accepted. + /// Stable, unique per agreement -- consumers dedup on (agreement_id, event_type) + /// since emission is at-least-once (an accepted event may be redelivered). + #[prost(string, tag = "6")] + pub agreement_id: ::prost::alloc::string::String, + /// 0x address of the Indexer who accepted the indexing agreement + #[prost(string, tag = "1")] + pub indexer: ::prost::alloc::string::String, + /// Timestamp of when the agreement was accepted + #[prost(uint64, tag = "2")] + pub accepted_at: u64, + /// 0x Hex of the onchain transaction for the accepted indexing agreement + #[prost(string, tag = "3")] + pub accepted_tx: ::prost::alloc::string::String, + /// Timestamp of when the agreement is scheduled to end + #[prost(uint64, tag = "4")] + pub ends_at: u64, + /// 0x address of the payer + #[prost(string, tag = "5")] + pub payer: ::prost::alloc::string::String, +} +/// SubgraphIndexingAgreementRequestExpired is emitted when an Indexer candidate fails to accept within the provided time limit +/// +/// Event type: subgraph.indexing.agreement.request.expired +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubgraphIndexingAgreementRequestExpired { + /// 0x hex of the on-chain agreement id (bytes16) that expired. + /// Stable, unique per agreement -- consumers dedup on (agreement_id, event_type) + /// since emission is at-least-once (an expired event may be redelivered). + #[prost(string, tag = "4")] + pub agreement_id: ::prost::alloc::string::String, + /// 0x address of the Indexer who failed to accept the agreement within the given time limit + #[prost(string, tag = "1")] + pub indexer: ::prost::alloc::string::String, + /// Timestamp of when the Indexing agreement was proposed + #[prost(uint64, tag = "2")] + pub request_proposed_at: u64, + /// Timestamp of when the required expired + #[prost(uint64, tag = "3")] + pub request_expired_at: u64, +} +/// SubgraphIndexingAgreementNIndexersUnavailable is emitted when the IISA selection returns fewer Indexer candidates than requested +/// +/// Event type: subgraph.indexing.agreement.n_indexers_unavailable +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubgraphIndexingAgreementNIndexersUnavailable { + /// Number of indexing agreements requested by the Subgraph developer. + #[prost(int32, tag = "1")] + pub agreements_requested: i32, + /// Number of indexer candidates selected by the IISA + #[prost(int32, tag = "2")] + pub candidates_returned: i32, +} +/// SubgraphIndexingAgreementTerminated is emitted if either the Subgraph developer, or Indexer, cancel the agreement +/// +/// Event type: subgraph.indexing.agreement.terminated +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SubgraphIndexingAgreementTerminated { + /// 0x hex of the on-chain agreement id (bytes16) that was terminated. + /// Stable, unique per agreement -- consumers dedup on (agreement_id, event_type) + /// since emission is at-least-once (a terminated event may be redelivered). + #[prost(string, tag = "6")] + pub agreement_id: ::prost::alloc::string::String, + /// 0x address of the Indexer that had the accepted agreement that was terminated + #[prost(string, tag = "1")] + pub indexer: ::prost::alloc::string::String, + /// Timestamp of when the Indexing Agreement was terminated + #[prost(uint64, tag = "2")] + pub terminated_at: u64, + /// 0x address of the entity (developer, indexer, etc) that terminated the agreement + #[prost(string, tag = "3")] + pub terminated_by: ::prost::alloc::string::String, + /// 0x hex of the onchain transaction of the agreement cancellation + #[prost(string, tag = "4")] + pub terminated_tx: ::prost::alloc::string::String, + /// Count of remaining active (accepted) agreements on the Subgraph, excluding + /// the one just terminated. + /// 0 -> the Subgraph no longer has any active, accepted indexing agreements. + /// -1 -> the count was unavailable when the event was emitted (e.g. a transient + /// datastore error); treat as unknown, not as zero. + #[prost(int32, tag = "5")] + pub remaining_accepted_indexing_agreements: i32, +} diff --git a/dipper-producer/src/proto/mod.rs b/dipper-producer/src/proto/mod.rs new file mode 100644 index 00000000..75c540c5 --- /dev/null +++ b/dipper-producer/src/proto/mod.rs @@ -0,0 +1,4 @@ +// Generated protobuf types for indexing agreement events +// +// To regenerate, run: just gen-indexing-agreement-events-proto +include!("dipper.subgraph.indexing.agreement.events.v1.rs"); diff --git a/justfile b/justfile index 6b2a5e98..70778005 100644 --- a/justfile +++ b/justfile @@ -112,3 +112,8 @@ remove-git-hooks: # Remove the pre-commit hooks pre-commit uninstall --config .github/pre-commit-config.yaml + +# Generate job events protobuf bindings (RUSTFLAGS="--cfg gen_event_proto" cargo check) +[group: 'codegen'] +gen-indexing-agreement-events-proto: + RUSTFLAGS="--cfg gen_event_proto" cargo check -p dipper-producer \ No newline at end of file