From fceda3cf32936e145fcd6f48e734687bca2ef9f6 Mon Sep 17 00:00:00 2001 From: Mark Kirichenko Date: Tue, 21 Jul 2026 09:20:48 +0200 Subject: [PATCH 1/2] feat(metrics): add Rust no_std encoder for V3 Add a hand-rolled Rust encoder for metrics V3 payload format. For performance and compatibility reasons we don't want to rely on a generated encoder (which also vendors the protobuf dependency), so we add the hand-rolled version and guardrails to ensure that we can catch changes in the `.proto` definition. Signed-off-by: Mark Kirichenko --- .github/workflows/test.yml | 64 + .gitignore | 3 + README.md | 12 +- REVIEWING.md | 2 + metrics/dd-metrics-v3/Cargo.lock | 614 ++++++++++ metrics/dd-metrics-v3/Cargo.toml | 30 + metrics/dd-metrics-v3/build.rs | 89 ++ metrics/dd-metrics-v3/src/constants.rs | 134 +++ metrics/dd-metrics-v3/src/interner.rs | 96 ++ metrics/dd-metrics-v3/src/lib.rs | 44 + metrics/dd-metrics-v3/src/types.rs | 257 ++++ metrics/dd-metrics-v3/src/writer.rs | 1488 ++++++++++++++++++++++++ metrics/dd-metrics-v3/tests/parity.rs | 833 +++++++++++++ metrics/dd-metrics-v3/tests/pb/mod.rs | 215 ++++ rust-toolchain.toml | 4 + 15 files changed, 3884 insertions(+), 1 deletion(-) create mode 100644 metrics/dd-metrics-v3/Cargo.lock create mode 100644 metrics/dd-metrics-v3/Cargo.toml create mode 100644 metrics/dd-metrics-v3/build.rs create mode 100644 metrics/dd-metrics-v3/src/constants.rs create mode 100644 metrics/dd-metrics-v3/src/interner.rs create mode 100644 metrics/dd-metrics-v3/src/lib.rs create mode 100644 metrics/dd-metrics-v3/src/types.rs create mode 100644 metrics/dd-metrics-v3/src/writer.rs create mode 100644 metrics/dd-metrics-v3/tests/parity.rs create mode 100644 metrics/dd-metrics-v3/tests/pb/mod.rs create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa67c9a3..dd2bb388 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,5 +52,69 @@ jobs: - run: inv codegen.all env: GOPATH: "/home/runner/work/agent-payload/agent-payload/go" + - name: Read Rust version from rust-toolchain.toml + id: rust-version + run: echo "version=$(grep -Po '^channel = "\K[^"]+' rust-toolchain.toml)" >> $GITHUB_OUTPUT + - name: Install ${{ steps.rust-version.outputs.version }} toolchain + run: | + rustup set profile minimal + rustup install ${{ steps.rust-version.outputs.version }} + rustup default ${{ steps.rust-version.outputs.version }} + # `inv codegen.all` (above) already ran `inv codegen.install-protoc`, so `metrics/dd-metrics-v3`'s + # build script (which reuses that same pinned protoc, see tasks/codegen.py's `protoc_version`) + # finds it without a separate install step. + - name: Regenerate metrics/dd-metrics-v3's protobuf test bindings + run: cargo build --features generate-protobuf + working-directory: "/home/runner/work/agent-payload/agent-payload/go/src/github.com/DataDog/agent-payload/metrics/dd-metrics-v3" - name: Check for diffs run: git diff --exit-code + + test-rust: + name: "cargo test / fmt / clippy (dd-metrics-v3)" + runs-on: ubuntu-latest + defaults: + run: + working-directory: metrics/dd-metrics-v3 + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - name: Read Rust version from rust-toolchain.toml + id: rust-version + run: echo "version=$(grep -Po '^channel = "\K[^"]+' ../../rust-toolchain.toml)" >> $GITHUB_OUTPUT + - name: Install ${{ steps.rust-version.outputs.version }} toolchain + run: | + rustup set profile minimal + rustup install ${{ steps.rust-version.outputs.version }} + rustup default ${{ steps.rust-version.outputs.version }} + rustup component add rustfmt clippy + - name: cargo fmt --check + run: cargo fmt -- --check + - name: cargo test + run: cargo test --all-targets + - name: cargo clippy + run: cargo clippy --all-targets -- -D warnings + + test-rust-no-std-check: + # `#![no_std]` alone only stops this crate's own code from referencing `std`; a dependency + # that quietly requires std would still link fine on a normal host target. To verify this, + # we build for a target with no OS where `std` is not available at all. + name: "dd-metrics-v3 builds for a target with no OS and no std" + runs-on: ubuntu-latest + defaults: + run: + working-directory: metrics/dd-metrics-v3 + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - name: Read Rust version from rust-toolchain.toml + id: rust-version + run: echo "version=$(grep -Po '^channel = "\K[^"]+' ../../rust-toolchain.toml)" >> $GITHUB_OUTPUT + - name: Install ${{ steps.rust-version.outputs.version }} toolchain + run: | + rustup set profile minimal + rustup install ${{ steps.rust-version.outputs.version }} + rustup default ${{ steps.rust-version.outputs.version }} + rustup target add x86_64-unknown-none + rustup component add clippy + - name: Build for no_std target + run: cargo build --target x86_64-unknown-none + - name: Clippy for no_std target + run: cargo clippy --target x86_64-unknown-none -- -D warnings diff --git a/.gitignore b/.gitignore index 430b2e61..74cffef4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ toolchains/ # python virtualenv for invoke tasks venv/ + +# rust build artifacts +target/ diff --git a/README.md b/README.md index 7230b091..7b05b4df 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Payload format description for communication between the Agent and the Datadog b This repository includes the protocol-buffer IDL used by the agent6 and agent7 to communicate with the Datadog backend. Those payloads are only supported by the V2 API endpoints. -The generated Go, and Java implementations are checked into this repository and can be used directly. Other consumers may copy the `.proto` files into their repository and generate their own bindings. +The generated Go, Java, and Rust implementations are checked into this repository and can be used directly. Other consumers may copy the `.proto` files into their repository and generate their own bindings. # Use @@ -25,6 +25,7 @@ You will need * Python (3+, CI builds with 3.12) * Go (at least the version in `go.mod`) * A checkout of this repository within a GOPATH (so, at `$GOPATH/src/github.com/DataDog/agent-payload`) + * Rust, only if you're working on [`metrics/dd-metrics-v3/`](./metrics/dd-metrics-v3) — install the toolchain version pinned in `rust-toolchain.toml`. # Payloads @@ -41,6 +42,14 @@ The metrics payload is defined in [`proto/metrics/agent_payload.proto`](./proto/ The following implementations are available: * Go (gogofast): [github.com/DataDog/agent-payload/gogen](https://pkg.go.dev/github.com/DataDog/agent-payload/gogen) +### Metrics V3 + +The V3 metrics payload is defined in [`proto/metrics/intake_v3.proto`](./proto/metrics/intake_v3.proto). +It uses a columnar layout with dictionary-based string deduplication instead of one message per +time series. The following implementations are available: + * Go (protoc-gen-go): [github.com/DataDog/agent-payload/v5/metrics/intake_v3](https://pkg.go.dev/github.com/DataDog/agent-payload/v5/metrics/intake_v3). + * Rust: [`dd-metrics-v3`](./metrics/dd-metrics-v3), a hand-rolled `no_std` encoder. + ## Process The process payload is defined in [`proto/process/agent.proto`](./proto/process/agent.proto). @@ -66,6 +75,7 @@ After updating the IDL you must: - Regenerate the code: `inv codegen.all`, invoke will use gimme to run the task command with the current defined go version - If you have indentation/newlines changes, run `inv codegen.all` with the same Go version as defined in `go.mod` +- If you changed `proto/metrics/intake_v3.proto`, also regenerate `metrics/dd-metrics-v3/tests/pb/mod.rs` (`cargo build --features generate-protobuf` from within `metrics/dd-metrics-v3/`) and update the hand-rolled encoder in `metrics/dd-metrics-v3/src/` to match, since it doesn't use a Protocol Buffers library - Create a new tag with the updated version of the payload # Publishing Changes diff --git a/REVIEWING.md b/REVIEWING.md index 8f4d2671..e2d8aede 100644 --- a/REVIEWING.md +++ b/REVIEWING.md @@ -8,6 +8,8 @@ This is a _work in progress_ and additions to this list are welcome. If any of the `.proto` files are changed, then the corresponding Go code should be regenerated in the same PR (`GOPATH=$(go env GOPATH) inv codegen.all`). +If `proto/metrics/intake_v3.proto` changes, also regenerate `metrics/dd-metrics-v3/tests/pb/mod.rs` (`cargo build --features generate-protobuf` from within `metrics/dd-metrics-v3/`) and update `metrics/dd-metrics-v3/src/constants.rs` and the hand-rolled encoder in `metrics/dd-metrics-v3/src/writer.rs` to match — they don't use a Protocol Buffers library, so they won't fail to compile just because the wire format changed underneath them. + Tests should be run locally before making a PR, and should pass in CI before a PR is merged. ## Implications for Security diff --git a/metrics/dd-metrics-v3/Cargo.lock b/metrics/dd-metrics-v3/Cargo.lock new file mode 100644 index 00000000..ee5746ef --- /dev/null +++ b/metrics/dd-metrics-v3/Cargo.lock @@ -0,0 +1,614 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bolero" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff44d278fc0062c95327087ed96b3d256906d1d8f579e534a3de8d6b386913a" +dependencies = [ + "bolero-afl", + "bolero-engine", + "bolero-generator", + "bolero-honggfuzz", + "bolero-kani", + "bolero-libfuzzer", + "cfg-if", + "rand", +] + +[[package]] +name = "bolero-afl" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9bf4cbd0bacf9356d3c7e5d9d088480f2076ba3c595c15ee9a6a378cdd7b297" +dependencies = [ + "bolero-engine", + "cc", +] + +[[package]] +name = "bolero-engine" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca199170a7c92c669c1019f9219a316b66bcdcfa4b36cac5a460a4c1a851aba" +dependencies = [ + "anyhow", + "bolero-generator", + "lazy_static", + "pretty-hex", + "rand", + "rand_xoshiro", +] + +[[package]] +name = "bolero-generator" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98a5782f2650f80d533f58ec339c6dce4cc5428f9c2755894f98156f52af81f2" +dependencies = [ + "bolero-generator-derive", + "either", + "getrandom", + "rand_core", + "rand_xoshiro", +] + +[[package]] +name = "bolero-generator-derive" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a21a3b022507b9edd2050caf370d945e398c1a7c8455531220fa3968c45d29e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bolero-honggfuzz" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a118ef27295eddefadc6a99728ee698d1b18d2e80dc4777d21bee3385096ffd" +dependencies = [ + "bolero-engine", +] + +[[package]] +name = "bolero-kani" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852ea5784a9f3e68bfd302ca80b8b863bce140593eb5770fee6ab110899c28fc" +dependencies = [ + "bolero-engine", +] + +[[package]] +name = "bolero-libfuzzer" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "858dc57c11725c52662501fa79fdbc6f7050339a05ca1bf1e587add0fed40d62" +dependencies = [ + "bolero-engine", + "cc", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "dd-metrics-v3" +version = "0.1.0" +dependencies = [ + "bolero", + "foldhash 0.2.0", + "hashbrown 0.16.1", + "prost", + "prost-build", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.187" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7743783ea728ef5c31194c6590797eed286449b4a4e87d626d8a51f0a94e732" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "pretty-hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a65843dfefbafd3c879c683306959a6de478443ffe9c9adf02f5976432402d7" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24" +dependencies = [ + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + +[[package]] +name = "toml_edit" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/metrics/dd-metrics-v3/Cargo.toml b/metrics/dd-metrics-v3/Cargo.toml new file mode 100644 index 00000000..355d8ded --- /dev/null +++ b/metrics/dd-metrics-v3/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "dd-metrics-v3" +version = "0.1.0" +description = "V3 columnar protobuf codec for Datadog metrics" +homepage = "https://github.com/DataDog/agent-payload/tree/master/metrics/dd-metrics-v3" +repository = "https://github.com/DataDog/agent-payload" +edition = "2021" +rust-version = "1.82.0" +license = "BSD-3-Clause" +authors = ["Datadog Inc. "] +autobenches = false + +[lib] +bench = false + +[dependencies] +# no `default-features` to keep it `no_std`. +foldhash = { version = "0.2", default-features = false } +hashbrown = { version = "0.16", default-features = false } + +[dev-dependencies] +bolero = "0.13" +prost = "0.14" + +[build-dependencies] +prost-build = { version = "0.14", optional = true } + +[features] +# Regenerates `tests/pb/mod.rs` from `../../proto/metrics/intake_v3.proto`. +generate-protobuf = ["dep:prost-build"] diff --git a/metrics/dd-metrics-v3/build.rs b/metrics/dd-metrics-v3/build.rs new file mode 100644 index 00000000..66417488 --- /dev/null +++ b/metrics/dd-metrics-v3/build.rs @@ -0,0 +1,89 @@ +// This build script always runs on the host with std available, so the workspace-wide +// std-vs-core/alloc and expect-used lints don't apply here. +#![allow( + clippy::std_instead_of_core, + clippy::std_instead_of_alloc, + clippy::expect_used +)] + +fn main() { + #[cfg(feature = "generate-protobuf")] + generate::run() + .expect("failed to regenerate protobuf bindings from proto/metrics/intake_v3.proto"); + #[cfg(not(feature = "generate-protobuf"))] + println!("cargo:rerun-if-changed=build.rs"); +} + +/// Regenerates `tests/pb/mod.rs` from `../../proto/metrics/intake_v3.proto`. Only compiled when the +/// `generate-protobuf` feature is enabled; normal builds and test runs use the checked-in +/// `tests/pb/mod.rs`. +#[cfg(feature = "generate-protobuf")] +mod generate { + use std::error::Error; + use std::{env, fs, path::Path}; + + const HEADER: &str = "// This file is @generated by prost-build from `proto/metrics/intake_v3.proto`. Do not edit it +// directly: regenerate it with `cargo build -p dd-metrics-v3 --features generate-protobuf` after +// changing the proto file. +#![allow(dead_code, clippy::all, clippy::pedantic, clippy::nursery)] + +"; + + /// Pinned by `tasks/codegen.py`'s `protoc_version`; keep in sync with that file so the + /// Go/Java and Rust bindings are generated by the exact same `protoc`. + const PROTOC_VERSION: &str = "21.12"; + + pub fn run() -> Result<(), Box> { + let cur_working_dir = env::var("CARGO_MANIFEST_DIR")?; + let crate_root = Path::new(&cur_working_dir); + let repo_root = crate_root.join("../.."); + + // Reuse the repo's own pinned protoc. + let protoc = repo_root.join(format!("toolchains/bin/protoc{PROTOC_VERSION}")); + if !protoc.exists() { + return Err(format!( + "protoc not found at {}; run `inv codegen.install-protoc` from the repo root first", + protoc.display() + ) + .into()); + } + env::set_var("PROTOC", &protoc); + + let toolchain_include_dir = repo_root.join("toolchains/include/proto"); + + let out_dir = crate_root.join("tests/pb"); + fs::create_dir_all(&out_dir)?; + + let proto_root = crate_root.join("../../proto/metrics"); + let proto_file = proto_root.join("intake_v3.proto"); + + let mut config = prost_build::Config::new(); + config.out_dir(&out_dir); + + println!("cargo:rerun-if-changed={}", proto_file.display()); + config.compile_protos(&[proto_file], &[proto_root, toolchain_include_dir])?; + + // prost-build names the output file after the proto's package + // (`datadoghq.api.metrics.v3.rs`); rename it to `mod.rs` so `tests/parity.rs` can + // pull it in as a plain submodule. + let generated = out_dir.join("datadoghq.api.metrics.v3.rs"); + let mod_rs = out_dir.join("mod.rs"); + fs::rename(&generated, &mod_rs)?; + + prepend_to_file(HEADER.as_bytes(), &mod_rs)?; + + Ok(()) + } + + fn prepend_to_file(data: &[u8], file_path: &Path) -> Result<(), Box> { + use std::io::{Read, Write}; + + let mut f = fs::File::open(file_path)?; + let mut content = data.to_owned(); + f.read_to_end(&mut content)?; + + let mut f = fs::File::create(file_path)?; + f.write_all(content.as_slice())?; + Ok(()) + } +} diff --git a/metrics/dd-metrics-v3/src/constants.rs b/metrics/dd-metrics-v3/src/constants.rs new file mode 100644 index 00000000..da63a206 --- /dev/null +++ b/metrics/dd-metrics-v3/src/constants.rs @@ -0,0 +1,134 @@ +// Protocol Buffers field numbers for the `MetricData` message in the V3 format. +// +// These field numbers come from the Protocol Buffers definitions in `proto/intake_v3.proto`, +// vendored from https://github.com/DataDog/agent-payload/blob/master/proto/metrics/intake_v3.proto. +/// Field number for the `DictNameStr` column. +pub const DICT_NAME_STR_FIELD_NUMBER: u32 = 1; +/// Field number for the `DictTagsStr` column. +pub const DICT_TAGS_STR_FIELD_NUMBER: u32 = 2; +/// Field number for the `DictTagsets` column. +pub const DICT_TAGSETS_FIELD_NUMBER: u32 = 3; +/// Field number for the `DictResourceStr` column. +pub const DICT_RESOURCE_STR_FIELD_NUMBER: u32 = 4; +/// Field number for the `DictResourcesLen` column. +pub const DICT_RESOURCE_LEN_FIELD_NUMBER: u32 = 5; +/// Field number for the `DictResourceType` column. +pub const DICT_RESOURCE_TYPE_FIELD_NUMBER: u32 = 6; +/// Field number for the `DictResourceName` column. +pub const DICT_RESOURCE_NAME_FIELD_NUMBER: u32 = 7; +/// Field number for the `DictSourceTypeName` column. +pub const DICT_SOURCE_TYPE_NAME_FIELD_NUMBER: u32 = 8; +/// Field number for the `DictOriginInfo` column. +pub const DICT_ORIGIN_INFO_FIELD_NUMBER: u32 = 9; +/// Field number for the `Type` column. +pub const TYPES_FIELD_NUMBER: u32 = 10; +/// Field number for the `Name` column. +pub const NAMES_FIELD_NUMBER: u32 = 11; +/// Field number for the `Tags` column. +pub const TAGS_FIELD_NUMBER: u32 = 12; +/// Field number for the `Resources` column. +pub const RESOURCES_FIELD_NUMBER: u32 = 13; +/// Field number for the `Interval` column. +pub const INTERVALS_FIELD_NUMBER: u32 = 14; +/// Field number for the `NumPoints` column. +pub const NUM_POINTS_FIELD_NUMBER: u32 = 15; +/// Field number for the `Timestamp` column. +pub const TIMESTAMPS_FIELD_NUMBER: u32 = 16; +/// Field number for the `ValueSint64` column. +pub const VALS_SINT64_FIELD_NUMBER: u32 = 17; +/// Field number for the `ValueFloat32` column. +pub const VALS_FLOAT32_FIELD_NUMBER: u32 = 18; +/// Field number for the `ValueFloat64` column. +pub const VALS_FLOAT64_FIELD_NUMBER: u32 = 19; +/// Field number for the `SketchNBins` column. +pub const SKETCH_NUM_BINS_FIELD_NUMBER: u32 = 20; +/// Field number for the `SketchBinKeys` column. +pub const SKETCH_BIN_KEYS_FIELD_NUMBER: u32 = 21; +/// Field number for the `SketchBinCounts` column. +pub const SKETCH_BIN_CNTS_FIELD_NUMBER: u32 = 22; +/// Field number for the `SourceTypeName` column. +pub const SOURCE_TYPE_NAME_FIELD_NUMBER: u32 = 23; +/// Field number for the `OriginInfo` column. +pub const ORIGIN_INFO_FIELD_NUMBER: u32 = 24; +/// Field number for the `DictUnitStr` column. +pub const DICT_UNIT_STR_FIELD_NUMBER: u32 = 25; +/// Field number for the `UnitRef` column. +pub const UNIT_REFS_FIELD_NUMBER: u32 = 26; + +/// Display names for the V3 columns, indexed by their Protocol Buffers field number. +/// +/// Field numbers come from `proto/intake_v3.proto`, also mirrored in `crate::writer` as the +/// `*_FIELD_NUMBER` constants. Index 0 is unused since field numbers start at 1. +pub const COLUMN_NAMES: [&str; 27] = [ + "reserved", + "DictNameStr", + "DictTagsStr", + "DictTagsets", + "DictResourceStr", + "DictResourcesLen", + "DictResourceType", + "DictResourceName", + "DictSourceTypeName", + "DictOriginInfo", + "Type", + "Name", + "Tags", + "Resources", + "Interval", + "NumPoints", + "Timestamp", + "ValueSint64", + "ValueFloat32", + "ValueFloat64", + "SketchNBins", + "SketchBinKeys", + "SketchBinCounts", + "SourceTypeName", + "OriginInfo", + "DictUnitStr", + "UnitRef", +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn field_numbers_match_column_names() { + let pairs: &[(u32, &str)] = &[ + (DICT_NAME_STR_FIELD_NUMBER, "DictNameStr"), + (DICT_TAGS_STR_FIELD_NUMBER, "DictTagsStr"), + (DICT_TAGSETS_FIELD_NUMBER, "DictTagsets"), + (DICT_RESOURCE_STR_FIELD_NUMBER, "DictResourceStr"), + (DICT_RESOURCE_LEN_FIELD_NUMBER, "DictResourcesLen"), + (DICT_RESOURCE_TYPE_FIELD_NUMBER, "DictResourceType"), + (DICT_RESOURCE_NAME_FIELD_NUMBER, "DictResourceName"), + (DICT_SOURCE_TYPE_NAME_FIELD_NUMBER, "DictSourceTypeName"), + (DICT_ORIGIN_INFO_FIELD_NUMBER, "DictOriginInfo"), + (TYPES_FIELD_NUMBER, "Type"), + (NAMES_FIELD_NUMBER, "Name"), + (TAGS_FIELD_NUMBER, "Tags"), + (RESOURCES_FIELD_NUMBER, "Resources"), + (INTERVALS_FIELD_NUMBER, "Interval"), + (NUM_POINTS_FIELD_NUMBER, "NumPoints"), + (TIMESTAMPS_FIELD_NUMBER, "Timestamp"), + (VALS_SINT64_FIELD_NUMBER, "ValueSint64"), + (VALS_FLOAT32_FIELD_NUMBER, "ValueFloat32"), + (VALS_FLOAT64_FIELD_NUMBER, "ValueFloat64"), + (SKETCH_NUM_BINS_FIELD_NUMBER, "SketchNBins"), + (SKETCH_BIN_KEYS_FIELD_NUMBER, "SketchBinKeys"), + (SKETCH_BIN_CNTS_FIELD_NUMBER, "SketchBinCounts"), + (SOURCE_TYPE_NAME_FIELD_NUMBER, "SourceTypeName"), + (ORIGIN_INFO_FIELD_NUMBER, "OriginInfo"), + (DICT_UNIT_STR_FIELD_NUMBER, "DictUnitStr"), + (UNIT_REFS_FIELD_NUMBER, "UnitRef"), + ]; + + for (field_number, expected_name) in pairs { + assert_eq!( + COLUMN_NAMES[*field_number as usize], *expected_name, + "field number {field_number} should index COLUMN_NAMES to \"{expected_name}\"" + ); + } + } +} diff --git a/metrics/dd-metrics-v3/src/interner.rs b/metrics/dd-metrics-v3/src/interner.rs new file mode 100644 index 00000000..d1d7830f --- /dev/null +++ b/metrics/dd-metrics-v3/src/interner.rs @@ -0,0 +1,96 @@ +//! Generic interning for dictionary deduplication. + +use alloc::borrow::ToOwned; +use core::{borrow::Borrow, hash::Hash}; + +type FastBuildHasher = foldhash::quality::RandomState; +type FastHashMap = hashbrown::HashMap; + +/// Generic interning structure for dictionary deduplication. +/// +/// Assigns unique 1-based IDs to values, returning the same ID for duplicate values. +/// ID 0 is reserved for "empty/none" in the V3 format. +#[derive(Debug)] +pub struct Interner { + index: FastHashMap, + last_id: i64, +} + +impl Default for Interner { + fn default() -> Self { + Self::new() + } +} + +impl Interner { + /// Creates a new empty interner. + pub fn new() -> Self { + Self { + index: FastHashMap::default(), + last_id: 0, + } + } + + /// Gets the ID for a key, inserting it if not present. + /// + /// Returns `(id, is_new)` where `is_new` is true if the key was newly inserted. + /// IDs are 1-based (0 is reserved for empty/none values). + pub fn get_or_insert(&mut self, key: &Q) -> (i64, bool) + where + K: Borrow, + Q: ToOwned + Hash + Eq + ?Sized, + { + if let Some(&id) = self.index.get(key) { + (id, false) + } else { + self.last_id += 1; + self.index.insert(key.to_owned(), self.last_id); + (self.last_id, true) + } + } + + /// Returns the number of interned values. + #[cfg(test)] + pub fn len(&self) -> usize { + self.index.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_interner_basic() { + let mut interner: Interner = Interner::new(); + + // First insertion returns ID 1 and is_new=true + let (id1, is_new1) = interner.get_or_insert("hello"); + assert_eq!(id1, 1); + assert!(is_new1); + + // Second insertion of same value returns same ID and is_new=false + let (id2, is_new2) = interner.get_or_insert("hello"); + assert_eq!(id2, 1); + assert!(!is_new2); + + // New value gets next ID + let (id3, is_new3) = interner.get_or_insert("world"); + assert_eq!(id3, 2); + assert!(is_new3); + + assert_eq!(interner.len(), 2); + } + + #[test] + fn test_interner_tuples() { + let mut interner: Interner<(i32, i32, i32)> = Interner::new(); + + let (id1, _) = interner.get_or_insert(&(1, 2, 3)); + let (id2, _) = interner.get_or_insert(&(1, 2, 3)); + let (id3, _) = interner.get_or_insert(&(4, 5, 6)); + + assert_eq!(id1, id2); + assert_ne!(id1, id3); + } +} diff --git a/metrics/dd-metrics-v3/src/lib.rs b/metrics/dd-metrics-v3/src/lib.rs new file mode 100644 index 00000000..0d1732fa --- /dev/null +++ b/metrics/dd-metrics-v3/src/lib.rs @@ -0,0 +1,44 @@ +//! V3 columnar protobuf codec for Datadog metrics. +//! +//! This crate implements the V3 format for Datadog metrics payloads which uses +//! a columnar layout with dictionary-based string deduplication for efficient encoding. +//! +//! [`V3Writer`] accumulates metrics one at a time via [`V3Writer::write`], then +//! [`V3Writer::into_columns`] produces the encoded columns. [`V3Writer::finalize`] +//! serializes those columns into a protobuf payload. +//! +//! Consumers with their own Protocol Buffers implementation can instead serialize [`V3EncodedData`] +//! directly using the `*_FIELD_NUMBER` constants. + +#![cfg_attr(not(test), no_std)] +#![deny(missing_docs)] +#![deny( + clippy::std_instead_of_core, + clippy::std_instead_of_alloc, + clippy::alloc_instead_of_core +)] + +extern crate alloc; + +mod constants; +mod interner; +mod types; +mod writer; + +pub use constants::{ + COLUMN_NAMES, DICT_NAME_STR_FIELD_NUMBER, DICT_ORIGIN_INFO_FIELD_NUMBER, + DICT_RESOURCE_LEN_FIELD_NUMBER, DICT_RESOURCE_NAME_FIELD_NUMBER, + DICT_RESOURCE_STR_FIELD_NUMBER, DICT_RESOURCE_TYPE_FIELD_NUMBER, + DICT_SOURCE_TYPE_NAME_FIELD_NUMBER, DICT_TAGSETS_FIELD_NUMBER, DICT_TAGS_STR_FIELD_NUMBER, + DICT_UNIT_STR_FIELD_NUMBER, INTERVALS_FIELD_NUMBER, NAMES_FIELD_NUMBER, + NUM_POINTS_FIELD_NUMBER, ORIGIN_INFO_FIELD_NUMBER, RESOURCES_FIELD_NUMBER, + SKETCH_BIN_CNTS_FIELD_NUMBER, SKETCH_BIN_KEYS_FIELD_NUMBER, SKETCH_NUM_BINS_FIELD_NUMBER, + SOURCE_TYPE_NAME_FIELD_NUMBER, TAGS_FIELD_NUMBER, TIMESTAMPS_FIELD_NUMBER, TYPES_FIELD_NUMBER, + UNIT_REFS_FIELD_NUMBER, VALS_FLOAT32_FIELD_NUMBER, VALS_FLOAT64_FIELD_NUMBER, + VALS_SINT64_FIELD_NUMBER, +}; +pub use types::V3MetricType; +pub use writer::{ + V3EncodedData, V3EncodedMetrics, V3EncoderStats, V3MetricBuilder, V3ValueEncodingStats, + V3Writer, V3WriterError, +}; diff --git a/metrics/dd-metrics-v3/src/types.rs b/metrics/dd-metrics-v3/src/types.rs new file mode 100644 index 00000000..ab6c5793 --- /dev/null +++ b/metrics/dd-metrics-v3/src/types.rs @@ -0,0 +1,257 @@ +//! V3 payload type definitions and protocol buffer field numbers. + +/// V3 metric type values. +/// +/// These match the `metricType` enum in `intake_v3.proto`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum V3MetricType { + /// A monotonically increasing counter, submitted as per-interval deltas. + Count = 1, + /// A count normalized to a per-second rate. + Rate = 2, + /// A point-in-time value. + Gauge = 3, + /// A distribution summarized as a `DDSketch`. + Sketch = 4, +} + +impl V3MetricType { + /// Returns the numeric value for encoding in the types column. + #[must_use] + pub const fn as_u64(self) -> u64 { + self as u64 + } +} + +/// V3 value type values. +/// +/// These are encoded in bits 4-7 of the types column and indicate which +/// value array contains the metric's points. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum V3ValueType { + /// Value is zero, not stored explicitly. + Zero = 0x00, + + /// Value is stored in `vals_sint64`. + Sint64 = 0x10, + + /// Value is stored in `vals_float32`. + Float32 = 0x20, + + /// Value is stored in `vals_float64`. + Float64 = 0x30, +} + +impl V3ValueType { + /// Returns the numeric value for encoding in the types column. + #[must_use] + pub const fn as_u64(self) -> u64 { + self as u64 + } +} + +/// Intermediate point classification for value type compaction. +/// +/// This provides finer-grained classification than [`V3ValueType`] to avoid +/// precision loss when combining different value types. In particular, it +/// distinguishes small integers (that fit losslessly in f32) from large integers +/// (that don't), so that mixing a large integer with a Float32 value correctly +/// escalates to Float64 rather than silently truncating the integer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +enum PointKind { + /// Value is zero. + Zero = 0, + /// Integer with |v| <= 2^24, fits losslessly in both sint64 and f32. + Int24 = 1, + /// Integer with |v| > 2^24, fits in sint64 varint but NOT losslessly in f32. + Int48 = 2, + /// Fractional value exactly representable as f32. + Float32 = 3, + /// Everything else - requires full f64 precision. + Float64 = 4, +} + +/// Maximum integer magnitude that fits losslessly in f32 (2^24). +const F32_INT_MAX: i64 = 1 << 24; + +impl PointKind { + /// Classifies a single f64 value. + // Casts round-trip `v` through a narrower type to check whether the conversion is lossless. + #[allow( + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + clippy::float_cmp + )] + fn for_value(v: f64) -> Self { + // Varint range that fits in 7 bytes or less (49 bits). + const VARINT_WIDTH: i32 = 7 * 7 - 1; + const MAX_INT: i64 = 1 << VARINT_WIDTH; + const MIN_INT: i64 = -MAX_INT; + + if v == 0.0 { + return Self::Zero; + } + + let i = v as i64; + if (MIN_INT..MAX_INT).contains(&i) && (i as f64) == v { + if (-F32_INT_MAX..=F32_INT_MAX).contains(&i) { + return Self::Int24; + } + return Self::Int48; + } + + if f64::from(v as f32) == v { + return Self::Float32; + } + + Self::Float64 + } + + /// Combines two point kinds into the smallest kind that can represent both. + /// + /// This is `max(self, other)` in all cases **except**: + /// - `Int48 + Float32 = Float64` (and vice versa), because large integers lose precision in + /// f32, and fractional values can't be stored as sint64. + fn union(self, other: Self) -> Self { + match (self, other) { + (Self::Int48, Self::Float32) | (Self::Float32, Self::Int48) => Self::Float64, + _ => self.max(other), + } + } + + /// Converts to the wire-format value type. + const fn to_value_type(self) -> V3ValueType { + match self { + Self::Zero => V3ValueType::Zero, + Self::Int24 | Self::Int48 => V3ValueType::Sint64, + Self::Float32 => V3ValueType::Float32, + Self::Float64 => V3ValueType::Float64, + } + } +} + +/// Determines the best [`V3ValueType`] for a set of f64 values. +/// +/// Uses [`PointKind`] internally to avoid precision loss when mixing +/// large integers with fractional float32 values. +pub fn value_type_for_values(values: impl Iterator) -> V3ValueType { + let mut kind = PointKind::Zero; + for v in values { + kind = kind.union(PointKind::for_value(v)); + } + kind.to_value_type() +} + +#[cfg(test)] +#[allow(clippy::cast_precision_loss, clippy::cast_lossless)] +mod tests { + use super::*; + + #[test] + fn test_point_kind_classification() { + // Zero + assert_eq!(PointKind::for_value(0.0), PointKind::Zero); + + // Small integers (fit in f32) + assert_eq!(PointKind::for_value(100.0), PointKind::Int24); + assert_eq!(PointKind::for_value(-100.0), PointKind::Int24); + assert_eq!(PointKind::for_value((1 << 24) as f64), PointKind::Int24); + assert_eq!(PointKind::for_value(-((1 << 24) as f64)), PointKind::Int24); + + // Large integers (don't fit losslessly in f32) + assert_eq!( + PointKind::for_value(((1 << 24) + 1) as f64), + PointKind::Int48 + ); + assert_eq!(PointKind::for_value((1i64 << 30) as f64), PointKind::Int48); + + // Float32 + assert_eq!(PointKind::for_value(1.5), PointKind::Float32); + assert_eq!(PointKind::for_value(2.75), PointKind::Float32); + + // Float64 + assert_eq!( + PointKind::for_value(core::f64::consts::PI), + PointKind::Float64 + ); + let large = ((1i64 << 50) + 1) as f64; + assert_eq!(PointKind::for_value(large), PointKind::Float64); + } + + #[test] + fn test_point_kind_union() { + // Standard widening (max) + assert_eq!(PointKind::Zero.union(PointKind::Int24), PointKind::Int24); + assert_eq!(PointKind::Int24.union(PointKind::Int48), PointKind::Int48); + assert_eq!( + PointKind::Int24.union(PointKind::Float32), + PointKind::Float32 + ); + assert_eq!( + PointKind::Float32.union(PointKind::Float64), + PointKind::Float64 + ); + assert_eq!( + PointKind::Float64.union(PointKind::Zero), + PointKind::Float64 + ); + + // The critical case: large integer + float32 must escalate to float64 + assert_eq!( + PointKind::Int48.union(PointKind::Float32), + PointKind::Float64 + ); + assert_eq!( + PointKind::Float32.union(PointKind::Int48), + PointKind::Float64 + ); + } + + #[test] + fn test_value_type_for_values() { + // All zeros + assert_eq!( + value_type_for_values([0.0, 0.0].into_iter()), + V3ValueType::Zero + ); + + // Small integers + assert_eq!( + value_type_for_values([100.0, 200.0].into_iter()), + V3ValueType::Sint64 + ); + + // Large integers + assert_eq!( + value_type_for_values([(1i64 << 30) as f64, 200.0].into_iter()), + V3ValueType::Sint64 + ); + + // Small integer + float32 → Float32 (safe, small int fits in f32) + assert_eq!( + value_type_for_values([100.0, 1.5].into_iter()), + V3ValueType::Float32 + ); + + // Large integer + float32 → Float64 (the bug fix!) + assert_eq!( + value_type_for_values([(1i64 << 30) as f64, 1.5].into_iter()), + V3ValueType::Float64 + ); + + // Float64 value forces Float64 + assert_eq!( + value_type_for_values([100.0, core::f64::consts::PI].into_iter()), + V3ValueType::Float64 + ); + + // Empty iterator + assert_eq!( + value_type_for_values(core::iter::empty()), + V3ValueType::Zero + ); + } +} diff --git a/metrics/dd-metrics-v3/src/writer.rs b/metrics/dd-metrics-v3/src/writer.rs new file mode 100644 index 00000000..dd3354fd --- /dev/null +++ b/metrics/dd-metrics-v3/src/writer.rs @@ -0,0 +1,1488 @@ +//! V3 columnar metrics writer. +//! +//! [`V3Writer`] accumulates metrics in columnar format with dictionary deduplication, then +//! produces [`V3EncodedData`] via [`V3Writer::into_columns`]. [`V3Writer::finalize`] +//! serializes that columnar data to protobuf wire format, using a hand-rolled encoder. + +use alloc::{string::String, vec::Vec}; + +use crate::{ + constants::{ + DICT_NAME_STR_FIELD_NUMBER, DICT_ORIGIN_INFO_FIELD_NUMBER, DICT_RESOURCE_LEN_FIELD_NUMBER, + DICT_RESOURCE_NAME_FIELD_NUMBER, DICT_RESOURCE_STR_FIELD_NUMBER, + DICT_RESOURCE_TYPE_FIELD_NUMBER, DICT_SOURCE_TYPE_NAME_FIELD_NUMBER, + DICT_TAGSETS_FIELD_NUMBER, DICT_TAGS_STR_FIELD_NUMBER, DICT_UNIT_STR_FIELD_NUMBER, + INTERVALS_FIELD_NUMBER, NAMES_FIELD_NUMBER, NUM_POINTS_FIELD_NUMBER, + ORIGIN_INFO_FIELD_NUMBER, RESOURCES_FIELD_NUMBER, SKETCH_BIN_CNTS_FIELD_NUMBER, + SKETCH_BIN_KEYS_FIELD_NUMBER, SKETCH_NUM_BINS_FIELD_NUMBER, SOURCE_TYPE_NAME_FIELD_NUMBER, + TAGS_FIELD_NUMBER, TIMESTAMPS_FIELD_NUMBER, TYPES_FIELD_NUMBER, UNIT_REFS_FIELD_NUMBER, + VALS_FLOAT32_FIELD_NUMBER, VALS_FLOAT64_FIELD_NUMBER, VALS_SINT64_FIELD_NUMBER, + }, + interner::Interner, + types::{value_type_for_values, V3MetricType, V3ValueType}, +}; + +pub const FLAG_NO_INDEX: u64 = 0x100; +pub const FLAG_HAS_UNIT: u64 = 0x200; + +/// Bitmask for the base [`V3MetricType`] stored in bits 0-3 of a `types` column entry. +const METRIC_TYPE_MASK: u64 = 0x0F; + +/// Errors returned by [`V3MetricBuilder`] methods when a caller violates one of their +/// preconditions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum V3WriterError { + /// [`add_point`](V3MetricBuilder::add_point) was called on a builder created with + /// [`V3MetricType::Sketch`]; use [`add_sketch`](V3MetricBuilder::add_sketch) instead. + PointOnSketchMetric, + /// [`add_sketch`](V3MetricBuilder::add_sketch) was called on a builder created with a + /// non-sketch metric type; use [`add_point`](V3MetricBuilder::add_point) instead. + SketchOnNonSketchMetric, + /// [`add_sketch`](V3MetricBuilder::add_sketch)'s `bin_keys` and `bin_counts` slices had + /// different lengths. + SketchBinLengthMismatch { + /// Length of the `bin_keys` slice. + bin_keys_len: usize, + /// Length of the `bin_counts` slice. + bin_counts_len: usize, + }, +} + +impl core::fmt::Display for V3WriterError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::PointOnSketchMetric => { + write!( + f, + "add_point called on a Sketch metric; use add_sketch instead" + ) + } + Self::SketchOnNonSketchMetric => { + write!( + f, + "add_sketch called on a non-Sketch metric; use add_point instead" + ) + } + Self::SketchBinLengthMismatch { + bin_keys_len, + bin_counts_len, + } => write!( + f, + "bin_keys length ({bin_keys_len}) does not match bin_counts length \ + ({bin_counts_len})" + ), + } + } +} + +impl core::error::Error for V3WriterError {} + +/// Encoded V3 payload data, prior to wire-format serialization. +/// +/// This is the columnar representation produced by [`V3Writer::into_columns`], after delta +/// encoding but before any protobuf framing. Column names are given by [`crate::COLUMN_NAMES`] +/// and field numbers by the `*_FIELD_NUMBER` constants in this crate. Consumers with their own +/// Protocol Buffers implementation can serialize these columns directly instead of using +/// [`V3Writer::finalize`]. +#[derive(Debug, Default)] +pub struct V3EncodedData { + /// Dictionary of interned metric names, as varint-length-prefixed strings concatenated + /// together. + pub dict_name_bytes: Vec, + /// Dictionary of interned tag strings, as varint-length-prefixed strings concatenated + /// together. + pub dict_tags_bytes: Vec, + /// Dictionary of interned tag sets. Each entry is a count of tags followed by that many + /// sorted, delta-encoded tag dictionary IDs. + pub dict_tagsets: Vec, + /// Dictionary of interned resource type/name strings, as varint-length-prefixed strings + /// concatenated together. + pub dict_resource_str_bytes: Vec, + /// Number of (type, name) pairs in each interned resource set. + pub dict_resource_len: Vec, + /// Delta-encoded resource type dictionary IDs, grouped contiguously per interned resource set. + pub dict_resource_type: Vec, + /// Delta-encoded resource name dictionary IDs, grouped contiguously per interned resource set. + pub dict_resource_name: Vec, + /// Dictionary of interned source type name strings, as varint-length-prefixed strings + /// concatenated together. + pub dict_source_type_bytes: Vec, + /// Dictionary of interned origin metadata, as flattened (product, category, service) triples. + pub dict_origin_info: Vec, + /// Dictionary of interned unit strings, as varint-length-prefixed strings concatenated + /// together. + pub dict_unit_bytes: Vec, + + /// Per-metric type and flags column. + pub types: Vec, + /// Per-metric delta-encoded name dictionary IDs. + pub names: Vec, + /// Per-metric delta-encoded tag set dictionary IDs. + pub tags: Vec, + /// Per-metric delta-encoded resource set dictionary IDs. + pub resources: Vec, + /// Per-metric interval, in seconds, used for rate metrics. + pub intervals: Vec, + /// Per-metric number of points. + pub num_points: Vec, + /// Per-metric delta-encoded source type name dictionary IDs. + pub source_type_names: Vec, + /// Per-metric delta-encoded origin metadata dictionary IDs. + pub origin_infos: Vec, + /// Per-metric delta-encoded unit dictionary IDs. Present only for metrics with `FLAG_HAS_UNIT` + /// set. + pub unit_refs: Vec, + + /// Delta-encoded point timestamps, across all metrics. + pub timestamps: Vec, + /// Point values stored as signed 64-bit integers. + pub vals_sint64: Vec, + /// Point values stored as 32-bit floats. + pub vals_float32: Vec, + /// Point values stored as 64-bit floats. + pub vals_float64: Vec, + + /// Number of bins in each sketch. + pub sketch_num_bins: Vec, + /// Delta-encoded sketch bin keys, grouped contiguously per sketch. + pub sketch_bin_keys: Vec, + /// Sketch bin counts, grouped contiguously per sketch. + pub sketch_bin_cnts: Vec, + + /// Telemetry produced while encoding the columns. + pub value_encoding_stats: V3ValueEncodingStats, +} + +/// Encoded V3 metrics payload with telemetry data. +pub struct V3EncodedMetrics { + /// Serialized `MetricData` protobuf payload. + pub payload: Vec, + /// Telemetry produced while encoding the payload. + pub stats: V3EncoderStats, +} + +/// Telemetry data produced while encoding a V3 metrics payload. +pub struct V3EncoderStats { + /// Counts of how many point values were compacted into each value column. + pub value_encoding_stats: V3ValueEncodingStats, + /// Raw bytes written for each present column, keyed by field number. + pub columns: Vec, +} + +/// Counts of how many point values were encoded into each value column. +#[derive(Clone, Copy, Debug, Default)] +pub struct V3ValueEncodingStats { + /// Number of point values that were zero and required no explicit storage. + pub zero: u64, + /// Number of point values stored as signed integers. + pub sint64: u64, + /// Number of point values stored as 32-bit floats. + pub float32: u64, + /// Number of point values stored as 64-bit floats. + pub float64: u64, +} + +/// Raw stream bytes for a single V3 column before protobuf field framing. +pub struct V3ColumnBytes { + /// Protocol Buffers field number this column corresponds to. + pub field_number: u32, + /// Column contents, framed as an unwrapped (no field tag) protobuf value. + pub bytes: Vec, + /// Reserved for the compressed length of `bytes`; currently always `0`. + pub compressed_len: usize, +} + +/// V3 columnar metrics writer. +/// +/// Accumulates metrics in columnar format with dictionary deduplication. +/// Call [`V3Writer::write`] for each metric, then [`V3Writer::finalize`] to finalize +/// and get the encoded data. +#[derive(Debug, Default)] +pub struct V3Writer { + // Interners for dictionary deduplication + name_interner: Interner, + tag_interner: Interner, + tagset_interner: Interner>, + resource_str_interner: Interner, + resource_interner: Interner>, + source_type_interner: Interner, + origin_interner: Interner<(i32, i32, i32)>, + unit_interner: Interner, + + // Dictionary encoded bytes + dict_name_bytes: Vec, + dict_tags_bytes: Vec, + dict_tagsets: Vec, + dict_resource_str_bytes: Vec, + dict_resource_len: Vec, + dict_resource_type: Vec, + dict_resource_name: Vec, + dict_source_type_bytes: Vec, + dict_origin_info: Vec, + dict_unit_bytes: Vec, + + // Per-metric columns (one entry per metric, except conditional columns) + types: Vec, + names: Vec, + tags: Vec, + resources: Vec, + intervals: Vec, + num_points: Vec, + source_type_names: Vec, + origin_infos: Vec, + unit_refs: Vec, // Present only for metrics with FLAG_HAS_UNIT set. + + // Point data + timestamps: Vec, + vals_sint64: Vec, + vals_float32: Vec, + vals_float64: Vec, + + // Sketch data + sketch_num_bins: Vec, + sketch_bin_keys: Vec, + sketch_bin_cnts: Vec, + + // Scratch data + tag_ids: Vec, + resource_ids: Vec<(i64, i64)>, + value_encoding_stats: V3ValueEncodingStats, +} + +impl V3Writer { + /// Creates a new V3 writer. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Begins writing a new metric. + /// + /// Returns a [`V3MetricBuilder`] that must be used to set the metric's + /// properties and add points, then closed with [`V3MetricBuilder::close`]. + pub fn write(&mut self, metric_type: V3MetricType, name: &str) -> V3MetricBuilder<'_> { + let name_id = self.intern_name(name); + let metric_idx = self.types.len(); + let point_start_idx = self.vals_float64.len(); + let sint64_start_idx = self.vals_sint64.len(); + + // Initialize the per-metric columns with default values + self.types.push(metric_type.as_u64()); + self.names.push(name_id); + self.tags.push(0); + self.resources.push(0); + self.intervals.push(0); + self.num_points.push(0); + self.source_type_names.push(0); + self.origin_infos.push(0); + + V3MetricBuilder { + writer: self, + point_start_idx, + sint64_start_idx, + metric_idx, + unit_ref_idx: None, + closed: false, + } + } + + /// Finalizes the writer and returns the encoded columnar data. + #[must_use] + pub fn into_columns(mut self) -> V3EncodedData { + // Delta encode all of the index arrays first. + delta_encode(&mut self.names); + delta_encode(&mut self.tags); + delta_encode(&mut self.resources); + delta_encode(&mut self.source_type_names); + delta_encode(&mut self.origin_infos); + delta_encode(&mut self.unit_refs); + delta_encode(&mut self.timestamps); + + V3EncodedData { + dict_name_bytes: self.dict_name_bytes, + dict_tags_bytes: self.dict_tags_bytes, + dict_tagsets: self.dict_tagsets, + dict_resource_str_bytes: self.dict_resource_str_bytes, + dict_resource_len: self.dict_resource_len, + dict_resource_type: self.dict_resource_type, + dict_resource_name: self.dict_resource_name, + dict_source_type_bytes: self.dict_source_type_bytes, + dict_origin_info: self.dict_origin_info, + dict_unit_bytes: self.dict_unit_bytes, + types: self.types, + names: self.names, + tags: self.tags, + resources: self.resources, + intervals: self.intervals, + num_points: self.num_points, + source_type_names: self.source_type_names, + origin_infos: self.origin_infos, + unit_refs: self.unit_refs, + timestamps: self.timestamps, + vals_sint64: self.vals_sint64, + vals_float32: self.vals_float32, + vals_float64: self.vals_float64, + sketch_num_bins: self.sketch_num_bins, + sketch_bin_keys: self.sketch_bin_keys, + sketch_bin_cnts: self.sketch_bin_cnts, + value_encoding_stats: self.value_encoding_stats, + } + } + + /// Finalizes the writer and serializes the data to the given output buffer. + #[allow(clippy::too_many_lines)] + pub fn finalize(self) -> V3EncodedMetrics { + let data = self.into_columns(); + let mut output = Vec::new(); + let mut columns = Vec::new(); + + // Dictionary fields (bytes - varint-length-prefixed strings concatenated) + write_bytes_column( + &mut output, + &mut columns, + DICT_NAME_STR_FIELD_NUMBER, + &data.dict_name_bytes, + ); + write_bytes_column( + &mut output, + &mut columns, + DICT_TAGS_STR_FIELD_NUMBER, + &data.dict_tags_bytes, + ); + + // Packed repeated fields for dictionaries + write_packed_column( + &mut output, + &mut columns, + DICT_TAGSETS_FIELD_NUMBER, + &data.dict_tagsets, + write_sint64_value, + ); + + write_bytes_column( + &mut output, + &mut columns, + DICT_RESOURCE_STR_FIELD_NUMBER, + &data.dict_resource_str_bytes, + ); + + write_packed_column( + &mut output, + &mut columns, + DICT_RESOURCE_LEN_FIELD_NUMBER, + &data.dict_resource_len, + write_int64_value, + ); + write_packed_column( + &mut output, + &mut columns, + DICT_RESOURCE_TYPE_FIELD_NUMBER, + &data.dict_resource_type, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + DICT_RESOURCE_NAME_FIELD_NUMBER, + &data.dict_resource_name, + write_sint64_value, + ); + + write_bytes_column( + &mut output, + &mut columns, + DICT_SOURCE_TYPE_NAME_FIELD_NUMBER, + &data.dict_source_type_bytes, + ); + + write_packed_column( + &mut output, + &mut columns, + DICT_ORIGIN_INFO_FIELD_NUMBER, + &data.dict_origin_info, + write_int32_value, + ); + write_bytes_column( + &mut output, + &mut columns, + DICT_UNIT_STR_FIELD_NUMBER, + &data.dict_unit_bytes, + ); + + // Per-metric columns + write_packed_column( + &mut output, + &mut columns, + TYPES_FIELD_NUMBER, + &data.types, + write_uint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + NAMES_FIELD_NUMBER, + &data.names, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + TAGS_FIELD_NUMBER, + &data.tags, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + RESOURCES_FIELD_NUMBER, + &data.resources, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + INTERVALS_FIELD_NUMBER, + &data.intervals, + write_uint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + NUM_POINTS_FIELD_NUMBER, + &data.num_points, + write_uint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + SOURCE_TYPE_NAME_FIELD_NUMBER, + &data.source_type_names, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + ORIGIN_INFO_FIELD_NUMBER, + &data.origin_infos, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + UNIT_REFS_FIELD_NUMBER, + &data.unit_refs, + write_sint64_value, + ); + + // Point data + write_packed_column( + &mut output, + &mut columns, + TIMESTAMPS_FIELD_NUMBER, + &data.timestamps, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + VALS_SINT64_FIELD_NUMBER, + &data.vals_sint64, + write_sint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + VALS_FLOAT32_FIELD_NUMBER, + &data.vals_float32, + write_f32_value, + ); + write_packed_column( + &mut output, + &mut columns, + VALS_FLOAT64_FIELD_NUMBER, + &data.vals_float64, + write_f64_value, + ); + + // Sketch data + write_packed_column( + &mut output, + &mut columns, + SKETCH_NUM_BINS_FIELD_NUMBER, + &data.sketch_num_bins, + write_uint64_value, + ); + write_packed_column( + &mut output, + &mut columns, + SKETCH_BIN_KEYS_FIELD_NUMBER, + &data.sketch_bin_keys, + write_sint32_value, + ); + write_packed_column( + &mut output, + &mut columns, + SKETCH_BIN_CNTS_FIELD_NUMBER, + &data.sketch_bin_cnts, + write_uint32_value, + ); + + V3EncodedMetrics { + payload: output, + stats: V3EncoderStats { + value_encoding_stats: data.value_encoding_stats, + columns, + }, + } + } + + fn intern_name(&mut self, name: &str) -> i64 { + if name.is_empty() { + return 0; + } + let (id, is_new) = self.name_interner.get_or_insert(name); + if is_new { + append_len_str(&mut self.dict_name_bytes, name); + } + id + } + + fn intern_tag(&mut self, tag: &str) { + if tag.is_empty() { + self.tag_ids.push(0); + return; + } + + let (id, is_new) = self.tag_interner.get_or_insert(tag); + if is_new { + append_len_str(&mut self.dict_tags_bytes, tag); + } + self.tag_ids.push(id); + } + + fn intern_tagset(&mut self, tags: I) -> i64 + where + I: Iterator, + S: AsRef, + { + self.tag_ids.clear(); + for tag in tags { + self.intern_tag(tag.as_ref()); + } + + if self.tag_ids.is_empty() { + return 0; + } + + let (id, is_new) = self.tagset_interner.get_or_insert(&self.tag_ids); + if is_new { + self.encode_tagset(); + } + id + } + + fn encode_tagset(&mut self) { + // Push the length. `tag_ids.len()` can never approach `i64::MAX`. + #[allow(clippy::cast_possible_wrap)] + self.dict_tagsets.push(self.tag_ids.len() as i64); + + let start = self.dict_tagsets.len(); + + // Add all tag IDs + self.dict_tagsets.extend_from_slice(&self.tag_ids); + + // Sort and delta-encode the tagset portion + self.dict_tagsets[start..].sort_unstable(); + delta_encode(&mut self.dict_tagsets[start..]); + } + + fn intern_resource_str(&mut self, s: &str) -> i64 { + if s.is_empty() { + return 0; + } + let (id, is_new) = self.resource_str_interner.get_or_insert(s); + if is_new { + append_len_str(&mut self.dict_resource_str_bytes, s); + } + id + } + + fn intern_resources(&mut self, resources: &[(&str, &str)]) -> i64 { + self.resource_ids.clear(); + for (resource_type, resource_name) in resources { + let type_id = self.intern_resource_str(resource_type); + let name_id = self.intern_resource_str(resource_name); + self.resource_ids.push((type_id, name_id)); + } + + if self.resource_ids.is_empty() { + return 0; + } + + let (id, is_new) = self.resource_interner.get_or_insert(&self.resource_ids); + if is_new { + self.encode_resources(); + } + id + } + + fn encode_resources(&mut self) { + // `resource_ids.len()` can never approach `i64::MAX`. + #[allow(clippy::cast_possible_wrap)] + self.dict_resource_len.push(self.resource_ids.len() as i64); + + let type_start = self.dict_resource_type.len(); + let name_start = self.dict_resource_name.len(); + + for (type_id, name_id) in &self.resource_ids { + self.dict_resource_type.push(*type_id); + self.dict_resource_name.push(*name_id); + } + + delta_encode(&mut self.dict_resource_type[type_start..]); + delta_encode(&mut self.dict_resource_name[name_start..]); + } + + fn intern_source_type(&mut self, s: &str) -> i64 { + if s.is_empty() { + return 0; + } + let (id, is_new) = self.source_type_interner.get_or_insert(s); + if is_new { + append_len_str(&mut self.dict_source_type_bytes, s); + } + id + } + + fn intern_origin(&mut self, product: i32, category: i32, service: i32) -> i64 { + if product == 0 && category == 0 && service == 0 { + return 0; + } + + let (id, is_new) = self + .origin_interner + .get_or_insert(&(product, category, service)); + if is_new { + self.dict_origin_info.push(product); + self.dict_origin_info.push(category); + self.dict_origin_info.push(service); + } + id + } + + fn intern_unit(&mut self, unit: &str) -> i64 { + if unit.is_empty() { + return 0; + } + let (id, is_new) = self.unit_interner.get_or_insert(unit); + if is_new { + append_len_str(&mut self.dict_unit_bytes, unit); + } + id + } +} + +/// Builder for a single metric within a V3 payload. +/// +/// Use the setter methods to configure the metric, add points with [`add_point`](Self::add_point), +/// then call [`close`](Self::close) to finalize. If dropped without calling `close`, the metric +/// is finalized automatically. +pub struct V3MetricBuilder<'a> { + writer: &'a mut V3Writer, + point_start_idx: usize, + sint64_start_idx: usize, + metric_idx: usize, + unit_ref_idx: Option, + closed: bool, +} + +impl Drop for V3MetricBuilder<'_> { + /// Finalizes the metric if [`close`](Self::close) was never called. + fn drop(&mut self) { + if !self.closed { + self.compact_values(); + } + } +} + +impl V3MetricBuilder<'_> { + /// Sets the tags for this metric. + /// + /// Tags should be in "key:value" format. + pub fn set_tags(&mut self, tags: I) + where + I: Iterator, + S: AsRef, + { + let tagset_id = self.writer.intern_tagset(tags); + self.writer.tags[self.metric_idx] = tagset_id; + } + + /// Sets the resources for this metric. + /// + /// Resources are (type, name) pairs, for example, (`host`, `server1`). + pub fn set_resources(&mut self, resources: &[(&str, &str)]) { + let res_id = self.writer.intern_resources(resources); + self.writer.resources[self.metric_idx] = res_id; + } + + /// Sets the interval for this metric (used for rate metrics). + pub fn set_interval(&mut self, interval: u64) { + self.writer.intervals[self.metric_idx] = interval; + } + + /// Sets the source type name for this metric. + pub fn set_source_type(&mut self, source_type: &str) { + if source_type.is_empty() { + self.writer.source_type_names[self.metric_idx] = 0; + return; + } + let id = self.writer.intern_source_type(source_type); + self.writer.source_type_names[self.metric_idx] = id; + } + + /// Sets the origin metadata for this metric. + #[allow(clippy::cast_possible_wrap)] + pub fn set_origin(&mut self, product: u32, category: u32, service: u32, no_index: bool) { + let id = self + .writer + .intern_origin(product as i32, category as i32, service as i32); + self.writer.origin_infos[self.metric_idx] = id; + if no_index { + self.writer.types[self.metric_idx] |= FLAG_NO_INDEX; + } else { + self.writer.types[self.metric_idx] &= !FLAG_NO_INDEX; + } + } + + /// Sets the unit for this metric. + pub fn set_unit(&mut self, unit: &str) { + if unit.is_empty() { + self.writer.types[self.metric_idx] &= !FLAG_HAS_UNIT; + if let Some(unit_ref_idx) = self.unit_ref_idx.take() { + self.writer.unit_refs.remove(unit_ref_idx); + } + return; + } + + let id = self.writer.intern_unit(unit); + if let Some(unit_ref_idx) = self.unit_ref_idx { + self.writer.unit_refs[unit_ref_idx] = id; + } else { + self.unit_ref_idx = Some(self.writer.unit_refs.len()); + self.writer.unit_refs.push(id); + } + self.writer.types[self.metric_idx] |= FLAG_HAS_UNIT; + } + + /// Adds a data point to this metric. + /// + /// # Errors + /// + /// Returns [`V3WriterError::PointOnSketchMetric`] if this builder was created with + /// [`V3MetricType::Sketch`]; use [`add_sketch`](Self::add_sketch) for sketch metrics instead. + pub fn add_point(&mut self, timestamp: i64, value: f64) -> Result<(), V3WriterError> { + if (self.writer.types[self.metric_idx] & METRIC_TYPE_MASK) == V3MetricType::Sketch as u64 { + return Err(V3WriterError::PointOnSketchMetric); + } + + self.writer.timestamps.push(timestamp); + self.writer.vals_float64.push(value); + self.writer.num_points[self.metric_idx] += 1; + Ok(()) + } + + /// Adds sketch data for a distribution metric. + /// + /// For sketches, the summary values (count, sum, min, max) are stored as points, + /// and the bin keys/counts are stored separately. + /// + /// # Errors + /// + /// Returns [`V3WriterError::SketchOnNonSketchMetric`] if this builder was not created with + /// [`V3MetricType::Sketch`], or [`V3WriterError::SketchBinLengthMismatch`] if `bin_keys` and + /// `bin_counts` have different lengths. + #[allow(clippy::too_many_arguments)] + pub fn add_sketch( + &mut self, + timestamp: i64, + count: i64, + sum: f64, + min: f64, + max: f64, + bin_keys: &[i32], + bin_counts: &[u32], + ) -> Result<(), V3WriterError> { + if (self.writer.types[self.metric_idx] & METRIC_TYPE_MASK) != V3MetricType::Sketch as u64 { + return Err(V3WriterError::SketchOnNonSketchMetric); + } + if bin_keys.len() != bin_counts.len() { + return Err(V3WriterError::SketchBinLengthMismatch { + bin_keys_len: bin_keys.len(), + bin_counts_len: bin_counts.len(), + }); + } + + self.writer.timestamps.push(timestamp); + + // Count goes in sint64, sum/min/max go in float64 + self.writer.vals_sint64.push(count); + self.writer.vals_float64.push(sum); + self.writer.vals_float64.push(min); + self.writer.vals_float64.push(max); + + // Store bin data + self.writer.sketch_num_bins.push(bin_keys.len() as u64); + + let key_start = self.writer.sketch_bin_keys.len(); + self.writer.sketch_bin_keys.extend_from_slice(bin_keys); + self.writer.sketch_bin_cnts.extend_from_slice(bin_counts); + + // Delta-encode this sketch's bin keys + delta_encode_i32(&mut self.writer.sketch_bin_keys[key_start..]); + + self.writer.num_points[self.metric_idx] += 1; + Ok(()) + } + + /// Finalizes this metric. + pub fn close(mut self) { + // Compacts the point values to use the smallest representation that can hold + // all values without loss. + self.compact_values(); + self.closed = true; + } + + #[allow(clippy::cast_possible_truncation)] + fn compact_values(&mut self) { + let count = self.writer.num_points[self.metric_idx] as usize; + if count == 0 { + return; + } + + let start = self.point_start_idx; + let end = self.writer.vals_float64.len(); + + // Determine the best value type for all points in this metric. + let val_ty = value_type_for_values(self.writer.vals_float64[start..end].iter().copied()); + let is_sketch = + (self.writer.types[self.metric_idx] & METRIC_TYPE_MASK) == V3MetricType::Sketch as u64; + let float_values_len = end - start; + if is_sketch { + // Sketches always carry one integer count per point in addition to sum/min/max values. + self.writer.value_encoding_stats.sint64 += count as u64; + } + + // Update the type field + self.writer.types[self.metric_idx] |= val_ty.as_u64(); + + // Convert values to the appropriate storage + match val_ty { + V3ValueType::Zero => { + self.writer.value_encoding_stats.zero += float_values_len as u64; + // Values are all zero, don't store anything + self.writer.vals_float64.truncate(start); + } + V3ValueType::Sint64 => { + self.writer.value_encoding_stats.sint64 += float_values_len as u64; + if is_sketch { + // For sketches, vals_sint64 already has one count per point (pushed by + // add_sketch), and vals_float64 has 3 values per point + // (sum, min, max). When compacting to Sint64, we need to + // interleave them as: sum, min, max, cnt per point. + let counts: Vec = + self.writer.vals_sint64[self.sint64_start_idx..].to_vec(); + self.writer.vals_sint64.truncate(self.sint64_start_idx); + for (i, cnt) in counts.into_iter().enumerate() { + let f_off = start + i * 3; + self.writer + .vals_sint64 + .push(self.writer.vals_float64[f_off] as i64); + self.writer + .vals_sint64 + .push(self.writer.vals_float64[f_off + 1] as i64); + self.writer + .vals_sint64 + .push(self.writer.vals_float64[f_off + 2] as i64); + self.writer.vals_sint64.push(cnt); + } + } else { + for i in start..end { + self.writer + .vals_sint64 + .push(self.writer.vals_float64[i] as i64); + } + } + self.writer.vals_float64.truncate(start); + } + V3ValueType::Float32 => { + self.writer.value_encoding_stats.float32 += float_values_len as u64; + for i in start..end { + self.writer + .vals_float32 + .push(self.writer.vals_float64[i] as f32); + } + self.writer.vals_float64.truncate(start); + } + V3ValueType::Float64 => { + self.writer.value_encoding_stats.float64 += float_values_len as u64; + // Already stored in vals_float64, keep them + } + } + } +} + +/// Protobuf wire type for length-delimited fields (bytes, strings, packed repeated fields). +const WIRE_LEN: u32 = 2; + +/// Writes a raw protobuf varint (LEB128, 7 bits per byte). +fn write_varint(buf: &mut Vec, mut value: u64) { + loop { + let byte = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + buf.push(byte); + return; + } + buf.push(byte | 0x80); + } +} + +/// Writes a protobuf field tag: `(field_number << 3) | wire_type`, as a varint. +fn write_tag(buf: &mut Vec, field_number: u32, wire_type: u32) { + write_varint(buf, (u64::from(field_number) << 3) | u64::from(wire_type)); +} + +/// Bit-reinterprets (not arithmetically converts) `v` as unsigned; this is the standard protobuf +/// zigzag transform, not a lossy narrowing. +#[allow(clippy::cast_sign_loss)] +const fn zigzag64(v: i64) -> u64 { + ((v << 1) ^ (v >> 63)) as u64 +} + +/// See [`zigzag64`]. +#[allow(clippy::cast_sign_loss)] +const fn zigzag32(v: i32) -> u32 { + ((v << 1) ^ (v >> 31)) as u32 +} + +// Scalar value encoders, one per protobuf field type used by the V3 payload. Each writes a +// single value with no tag or length prefix, matching the layout of a packed repeated field's +// payload (and reused as-is for `V3ColumnBytes::bytes`, which is exactly that payload). + +fn write_uint64_value(buf: &mut Vec, v: u64) { + write_varint(buf, v); +} + +fn write_sint64_value(buf: &mut Vec, v: i64) { + write_varint(buf, zigzag64(v)); +} + +/// Protobuf's `int64` wire type is a plain varint of the value's two's-complement bit pattern +/// (inefficient for negative numbers, but that's the wire format). +#[allow(clippy::cast_sign_loss)] +fn write_int64_value(buf: &mut Vec, v: i64) { + write_varint(buf, v as u64); +} + +/// See [`write_int64_value`]; `int32` is sign-extended to 64 bits before the same treatment. +#[allow(clippy::cast_sign_loss)] +fn write_int32_value(buf: &mut Vec, v: i32) { + write_varint(buf, i64::from(v) as u64); +} + +fn write_sint32_value(buf: &mut Vec, v: i32) { + write_varint(buf, u64::from(zigzag32(v))); +} + +fn write_uint32_value(buf: &mut Vec, v: u32) { + write_varint(buf, u64::from(v)); +} + +fn write_f32_value(buf: &mut Vec, v: f32) { + buf.extend_from_slice(&v.to_le_bytes()); +} + +fn write_f64_value(buf: &mut Vec, v: f64) { + buf.extend_from_slice(&v.to_le_bytes()); +} + +/// Writes a `bytes` field (tag + varint length + data). No-op if `bytes` is empty. +fn write_bytes_column( + output: &mut Vec, + columns: &mut Vec, + field_number: u32, + bytes: &[u8], +) { + if bytes.is_empty() { + return; + } + + write_tag(output, field_number, WIRE_LEN); + write_varint(output, bytes.len() as u64); + output.extend_from_slice(bytes); + + columns.push(V3ColumnBytes { + field_number, + bytes: bytes.to_vec(), + compressed_len: 0, + }); +} + +/// Writes a packed repeated field (tag + varint byte-length + concatenated encoded values). +/// No-op if `values` is empty. +fn write_packed_column( + output: &mut Vec, + columns: &mut Vec, + field_number: u32, + values: &[T], + encode_one: fn(&mut Vec, T), +) { + if values.is_empty() { + return; + } + + let mut raw = Vec::new(); + for &v in values { + encode_one(&mut raw, v); + } + + write_tag(output, field_number, WIRE_LEN); + write_varint(output, raw.len() as u64); + output.extend_from_slice(&raw); + + columns.push(V3ColumnBytes { + field_number, + bytes: raw, + compressed_len: 0, + }); +} + +fn append_len_str(dst: &mut Vec, s: &str) { + let mut len = s.len() as u64; + loop { + let mut byte = (len & 0x7F) as u8; + len >>= 7; + if len != 0 { + byte |= 0x80; + } + dst.push(byte); + if len == 0 { + break; + } + } + dst.extend_from_slice(s.as_bytes()); +} + +fn delta_encode(s: &mut [i64]) { + if s.len() < 2 { + return; + } + for i in (1..s.len()).rev() { + s[i] -= s[i - 1]; + } +} + +fn delta_encode_i32(s: &mut [i32]) { + if s.len() < 2 { + return; + } + for i in (1..s.len()).rev() { + s[i] -= s[i - 1]; + } +} + +#[cfg(test)] +#[allow(clippy::cast_precision_loss, clippy::cast_lossless)] +mod tests { + use super::*; + + #[test] + fn test_delta_encode() { + let mut data = vec![100, 110, 130, 145]; + delta_encode(&mut data); + assert_eq!(data, vec![100, 10, 20, 15]); + } + + #[test] + fn test_delta_encode_empty() { + let mut data: Vec = vec![]; + delta_encode(&mut data); + assert!(data.is_empty()); + } + + #[test] + fn test_delta_encode_single() { + let mut data = vec![42]; + delta_encode(&mut data); + assert_eq!(data, vec![42]); + } + + #[test] + fn test_append_len_str() { + let mut buf = Vec::new(); + append_len_str(&mut buf, "hello"); + // Length 5 = 0x05, then "hello" + assert_eq!(buf, vec![5, b'h', b'e', b'l', b'l', b'o']); + } + + #[test] + fn test_varint_encoding() { + let mut buf = Vec::new(); + write_varint(&mut buf, 0); + assert_eq!(buf, [0x00]); + buf.clear(); + write_varint(&mut buf, 127); + assert_eq!(buf, [0x7f]); + buf.clear(); + write_varint(&mut buf, 128); + assert_eq!(buf, [0x80, 0x01]); + buf.clear(); + write_varint(&mut buf, 300); + assert_eq!(buf, [0xac, 0x02]); + } + + #[test] + fn test_zigzag64() { + assert_eq!(zigzag64(0), 0); + assert_eq!(zigzag64(-1), 1); + assert_eq!(zigzag64(1), 2); + assert_eq!(zigzag64(-2), 3); + assert_eq!(zigzag64(2147483647), 4294967294); + assert_eq!(zigzag64(-2147483648), 4294967295); + } + + #[test] + fn test_zigzag32() { + assert_eq!(zigzag32(0), 0); + assert_eq!(zigzag32(-1), 1); + assert_eq!(zigzag32(1), 2); + assert_eq!(zigzag32(-2), 3); + } + + #[test] + fn test_write_f32_value_little_endian() { + let mut buf = Vec::new(); + write_f32_value(&mut buf, 1.0); + // 1.0f32 = 0x3f800000; LE bytes = [0x00, 0x00, 0x80, 0x3f] + assert_eq!(buf, [0x00, 0x00, 0x80, 0x3f]); + } + + #[test] + fn test_write_f64_value_little_endian() { + let mut buf = Vec::new(); + write_f64_value(&mut buf, 1.0); + assert_eq!(buf, 1.0f64.to_le_bytes()); + } + + #[test] + fn test_writer_basic() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "test.metric"); + metric.set_tags(["env:prod", "service:web"].iter().copied()); + metric.add_point(1000, 42.0).unwrap(); + metric.add_point(1010, 43.5).unwrap(); + metric.close(); + } + + let data = writer.into_columns(); + + assert_eq!(data.types.len(), 1); + assert_eq!(data.names.len(), 1); + assert_eq!(data.timestamps.len(), 2); + } + + #[test] + fn test_writer_unit() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "has.unit"); + metric.set_unit("millisecond"); + metric.add_point(1000, 42.0).unwrap(); + metric.close(); + } + { + let mut metric = writer.write(V3MetricType::Gauge, "no.unit"); + metric.add_point(1000, 43.0).unwrap(); + metric.close(); + } + { + let mut metric = writer.write(V3MetricType::Gauge, "same.unit"); + metric.set_unit("millisecond"); + metric.add_point(1000, 44.0).unwrap(); + metric.close(); + } + + let data = writer.into_columns(); + + assert_eq!(data.unit_refs, vec![1, 0]); + assert_eq!(data.dict_unit_bytes, b"\x0bmillisecond"); + assert_eq!(data.types[0] & FLAG_HAS_UNIT, FLAG_HAS_UNIT); + assert_eq!(data.types[1] & FLAG_HAS_UNIT, 0); + assert_eq!(data.types[2] & FLAG_HAS_UNIT, FLAG_HAS_UNIT); + } + + #[test] + fn test_writer_multiple_metrics() { + let mut writer = V3Writer::new(); + + { + let mut m1 = writer.write(V3MetricType::Count, "metric1"); + m1.add_point(1000, 10.0).unwrap(); + m1.close(); + } + + { + let mut m2 = writer.write(V3MetricType::Rate, "metric2"); + m2.set_interval(60); + m2.add_point(2000, 20.0).unwrap(); + m2.close(); + } + + let data = writer.into_columns(); + + assert_eq!(data.types.len(), 2); + assert_eq!(data.names.len(), 2); + assert_eq!(data.intervals[0], 0); + // Second metric's interval won't be 60 directly since names is delta-encoded, + // but we can verify the structure is correct + } + + #[test] + fn test_value_compaction_zero() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "zero.metric"); + metric.add_point(1000, 0.0).unwrap(); + metric.add_point(2000, 0.0).unwrap(); + metric.close(); + } + + let data = writer.into_columns(); + + // Values should be compacted - zero values don't need storage + assert!(data.vals_float64.is_empty()); + assert!(data.vals_sint64.is_empty()); + assert!(data.vals_float32.is_empty()); + } + + #[test] + fn test_value_compaction_int() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Count, "int.metric"); + metric.add_point(1000, 100.0).unwrap(); + metric.add_point(2000, 200.0).unwrap(); + metric.close(); + } + + let data = writer.into_columns(); + + // Integer values should be stored in sint64 + assert!(data.vals_float64.is_empty()); + assert_eq!(data.vals_sint64, vec![100, 200]); + assert!(data.vals_float32.is_empty()); + } + + #[test] + fn test_serialize_empty() { + let writer = V3Writer::new(); + let encoded = writer.finalize(); + assert!(encoded.payload.is_empty()); + } + + #[test] + fn test_value_compaction_large_int_plus_float32() { + // Regression test: a large integer (> 2^24) mixed with a fractional + // float32 value must use Float64, not Float32, to avoid precision loss. + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "mixed.metric"); + metric.add_point(1000, (1i64 << 30) as f64).unwrap(); // large int, doesn't fit in f32 + metric.add_point(2000, 1.5).unwrap(); // fractional, fits in f32 + metric.close(); + } + + let data = writer.into_columns(); + + // Must be stored in float64, not float32 + assert!( + data.vals_float32.is_empty(), + "large int should not be stored as float32" + ); + assert_eq!(data.vals_float64, vec![(1i64 << 30) as f64, 1.5]); + assert!(data.vals_sint64.is_empty()); + } + + #[test] + fn test_value_compaction_small_int_plus_float32() { + // Small integers (|v| <= 2^24) mixed with float32 values should + // compact to Float32, since small ints fit losslessly in f32. + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "small.mixed"); + metric.add_point(1000, 100.0).unwrap(); + metric.add_point(2000, 1.5).unwrap(); + metric.close(); + } + + let data = writer.into_columns(); + + assert!(data.vals_float64.is_empty()); + assert_eq!(data.vals_float32, vec![100.0, 1.5]); + assert!(data.vals_sint64.is_empty()); + } + + #[test] + fn test_serialize_basic_metric() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "test.metric"); + metric.add_point(1000, 42.0).unwrap(); + metric.close(); + } + + let encoded = writer.finalize(); + + // Should produce non-empty output + assert!(!encoded.payload.is_empty()); + assert_eq!(encoded.stats.value_encoding_stats.sint64, 1); + } + + #[test] + fn test_column_stats_for_bytes_column_use_raw_column_stream() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "test.metric"); + metric.add_point(1000, 42.0).unwrap(); + metric.close(); + } + + let encoded = writer.finalize(); + let name_column = encoded + .stats + .columns + .iter() + .find(|column| column.field_number == DICT_NAME_STR_FIELD_NUMBER) + .expect("name dictionary column should be present"); + + let mut expected = Vec::new(); + append_len_str(&mut expected, "test.metric"); + assert_eq!(name_column.bytes, expected); + } + + #[test] + fn test_column_stats_for_packed_column_use_raw_column_stream() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "test.metric"); + metric.add_point(1000, 42.0).unwrap(); + metric.close(); + } + + let encoded = writer.finalize(); + let timestamps_column = encoded + .stats + .columns + .iter() + .find(|column| column.field_number == TIMESTAMPS_FIELD_NUMBER) + .expect("timestamp column should be present"); + + let mut expected = Vec::new(); + write_sint64_value(&mut expected, 1000); + assert_eq!(timestamps_column.bytes, expected); + } + + #[test] + fn test_column_stats_do_not_include_absent_columns() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "test.metric"); + metric.add_point(1000, 42.0).unwrap(); + metric.close(); + } + + let encoded = writer.finalize(); + assert!(!encoded + .stats + .columns + .iter() + .any(|column| column.field_number == UNIT_REFS_FIELD_NUMBER)); + } + + #[test] + fn test_set_origin_clears_no_index_flag_when_reset() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "origin.metric"); + metric.set_origin(1, 2, 3, true); + metric.set_origin(1, 2, 3, false); + metric.add_point(1000, 1.0).unwrap(); + metric.close(); + } + + let data = writer.into_columns(); + assert_eq!(data.types[0] & FLAG_NO_INDEX, 0); + } + + #[test] + fn test_builder_finalizes_on_drop_without_close() { + let mut writer = V3Writer::new(); + + { + let mut metric = writer.write(V3MetricType::Gauge, "dropped.metric"); + metric.add_point(1000, 42.0).unwrap(); + // Deliberately not calling `close`; `Drop` must finalize the metric anyway. + } + + let data = writer.into_columns(); + assert_eq!(data.vals_sint64, vec![42]); + assert!(data.vals_float64.is_empty()); + assert_eq!(data.types[0] & 0x30, V3ValueType::Sint64.as_u64()); + } + + #[test] + fn test_add_point_rejected_on_sketch_metric() { + let mut writer = V3Writer::new(); + let mut metric = writer.write(V3MetricType::Sketch, "wrong.method"); + assert_eq!( + metric.add_point(1000, 1.0), + Err(V3WriterError::PointOnSketchMetric) + ); + } + + #[test] + fn test_add_sketch_rejected_on_non_sketch_metric() { + let mut writer = V3Writer::new(); + let mut metric = writer.write(V3MetricType::Gauge, "wrong.method"); + assert_eq!( + metric.add_sketch(1000, 1, 1.0, 1.0, 1.0, &[0], &[1]), + Err(V3WriterError::SketchOnNonSketchMetric) + ); + } + + #[test] + fn test_add_sketch_rejected_on_bin_length_mismatch() { + let mut writer = V3Writer::new(); + let mut metric = writer.write(V3MetricType::Sketch, "mismatched.bins"); + assert_eq!( + metric.add_sketch(1000, 1, 1.0, 1.0, 1.0, &[0, 1], &[1]), + Err(V3WriterError::SketchBinLengthMismatch { + bin_keys_len: 2, + bin_counts_len: 1, + }) + ); + } +} diff --git a/metrics/dd-metrics-v3/tests/parity.rs b/metrics/dd-metrics-v3/tests/parity.rs new file mode 100644 index 00000000..840c1c70 --- /dev/null +++ b/metrics/dd-metrics-v3/tests/parity.rs @@ -0,0 +1,833 @@ +//! Wire-format parity tests. +//! +//! `dd-metrics-v3` hand-rolls its own protobuf wire-format encoder (see `src/writer.rs`) +//! instead of depending on a Protocol Buffers library for efficiency purposes and to keep this +//! crate `no_std`. We check the encoder agains generated bindings in two dimensions: +//! +//! - [`assert_wire_parity`] checks *wire framing*: it decodes the hand-rolled encoder's bytes with +//! `prost`'s generated decoder and checks the result is exactly the [`V3EncodedData`] columns we +//! intended to encode. +//! - [`assert_points_round_trip`] checks *columnar correctness*: it decodes point/sketch values +//! straight from the wire bytes' raw `types`/`timestamps`/`vals_*`/`sketch_*` arrays using its +//! own delta-decoding and value-type dispatch that shares no code with `V3Writer`, and compares +//! the result against the exact values that were passed to `add_point`/`add_sketch`. + +// To check the encoder's value-type compaction boundaries +#![allow(clippy::cast_precision_loss)] +// For reference decoder check +#![allow(clippy::expect_used)] + +mod pb; + +use dd_metrics_v3::{V3EncodedData, V3MetricType, V3Writer}; +use prost::Message as _; + +/// Builds a [`pb::MetricData`] holding the exact same columnar data as `data`. +fn to_reference_message(data: V3EncodedData) -> pb::MetricData { + pb::MetricData { + dict_name_str: data.dict_name_bytes, + dict_tag_str: data.dict_tags_bytes, + dict_tagsets: data.dict_tagsets, + dict_resource_str: data.dict_resource_str_bytes, + dict_resource_len: data.dict_resource_len, + dict_resource_type: data.dict_resource_type, + dict_resource_name: data.dict_resource_name, + dict_source_type_name: data.dict_source_type_bytes, + dict_origin_info: data.dict_origin_info, + dict_unit_str: data.dict_unit_bytes, + types: data.types, + name_refs: data.names, + tagset_refs: data.tags, + resources_refs: data.resources, + intervals: data.intervals, + num_points: data.num_points, + source_type_name_refs: data.source_type_names, + origin_info_refs: data.origin_infos, + unit_refs: data.unit_refs, + timestamps: data.timestamps, + vals_sint64: data.vals_sint64, + vals_float32: data.vals_float32, + vals_float64: data.vals_float64, + sketch_num_bins: data.sketch_num_bins, + sketch_bin_keys: data.sketch_bin_keys, + sketch_bin_cnts: data.sketch_bin_cnts, + } +} + +/// Runs `build` against two independent [`V3Writer`]s: one is encoded with the hand-rolled encoder +/// under test, the other becomes the expected [`pb::MetricData`] via [`to_reference_message`]. +/// Asserts that decoding the hand-rolled bytes with `prost`'s generated decoder reproduces the +/// expected message exactly, and that the two encoded lengths match (see the module docs for why +/// this checks decoded equality plus length rather than raw byte equality). +#[track_caller] +fn assert_wire_parity(build: impl Fn(&mut V3Writer)) { + let mut ours = V3Writer::new(); + build(&mut ours); + let ours_bytes = ours.finalize().payload; + + let mut reference = V3Writer::new(); + build(&mut reference); + let expected = to_reference_message(reference.into_columns()); + + let decoded = pb::MetricData::decode(ours_bytes.as_slice()) + .expect("hand-rolled encoder output must be valid protobuf wire format"); + assert_eq!( + decoded, expected, + "decoding the hand-rolled encoder's bytes must reproduce the intended message" + ); + assert_eq!( + ours_bytes.len(), + expected.encoded_len(), + "hand-rolled and prost-encoded payloads must be the same length" + ); +} + +/// A point or sketch value decoded independently of `V3Writer`, identified by the bit pattern of +/// its f64 value (so NaN, which is unequal to itself under `==`, still compares correctly). +#[derive(Debug, Clone, PartialEq, Eq)] +enum DecodedPoint { + Value { + timestamp: i64, + bits: u64, + }, + Sketch { + timestamp: i64, + count: i64, + sum_bits: u64, + min_bits: u64, + max_bits: u64, + bin_keys: Vec, + bin_counts: Vec, + }, +} + +/// Decodes the point/sketch values of every metric in `data`, in writer-call order, straight from +/// the raw `types`/`timestamps`/`vals_*`/`sketch_*` columns — using this function's own +/// delta-decoding and value-type dispatch rather than anything from `src/writer.rs` or +/// `src/types.rs`. This is deliberately independent of `V3Writer` so that a bug in its +/// delta-encoding or value-type compaction changes this function's output without also changing +/// what it's compared against (see the module docs). +#[allow( + clippy::cast_sign_loss, + clippy::cast_possible_truncation, + clippy::too_many_lines, + clippy::panic +)] +fn decode_points(data: &pb::MetricData) -> Vec> { + const METRIC_TYPE_MASK: u64 = 0x0F; + const VALUE_TYPE_MASK: u64 = 0xF0; + const METRIC_TYPE_SKETCH: u64 = 4; + + fn delta_decode(values: &mut [i64]) { + for i in 1..values.len() { + values[i] += values[i - 1]; + } + } + + fn delta_decode_i32(values: &mut [i32]) { + for i in 1..values.len() { + values[i] += values[i - 1]; + } + } + + fn read_value( + data: &pb::MetricData, + value_type: u64, + sint_cursor: &mut usize, + f32_cursor: &mut usize, + f64_cursor: &mut usize, + ) -> f64 { + match value_type { + 0x00 => 0.0, + 0x10 => { + let v = data.vals_sint64[*sint_cursor]; + *sint_cursor += 1; + v as f64 + } + 0x20 => { + let v = data.vals_float32[*f32_cursor]; + *f32_cursor += 1; + f64::from(v) + } + 0x30 => { + let v = data.vals_float64[*f64_cursor]; + *f64_cursor += 1; + v + } + other => panic!("unknown v3 value type {other:#x}"), + } + } + + let mut timestamps = data.timestamps.clone(); + delta_decode(&mut timestamps); + let mut sketch_bin_keys = data.sketch_bin_keys.clone(); + + let (mut ts_cursor, mut sint_cursor, mut f32_cursor, mut f64_cursor) = (0, 0, 0, 0); + let (mut sketch_point_cursor, mut sketch_key_cursor, mut sketch_cnt_cursor) = (0, 0, 0); + + data.types + .iter() + .enumerate() + .map(|(i, &type_field)| { + let metric_type = type_field & METRIC_TYPE_MASK; + let value_type = type_field & VALUE_TYPE_MASK; + let num_points = data.num_points[i] as usize; + + (0..num_points) + .map(|_| { + let timestamp = timestamps[ts_cursor]; + ts_cursor += 1; + + if metric_type == METRIC_TYPE_SKETCH { + let sum = read_value( + data, + value_type, + &mut sint_cursor, + &mut f32_cursor, + &mut f64_cursor, + ); + let min = read_value( + data, + value_type, + &mut sint_cursor, + &mut f32_cursor, + &mut f64_cursor, + ); + let max = read_value( + data, + value_type, + &mut sint_cursor, + &mut f32_cursor, + &mut f64_cursor, + ); + let count = data.vals_sint64[sint_cursor]; + sint_cursor += 1; + + let num_bins = data.sketch_num_bins[sketch_point_cursor] as usize; + sketch_point_cursor += 1; + + let start = sketch_key_cursor; + delta_decode_i32(&mut sketch_bin_keys[start..start + num_bins]); + let bin_keys = sketch_bin_keys[start..start + num_bins].to_vec(); + sketch_key_cursor += num_bins; + + let bin_counts = data.sketch_bin_cnts + [sketch_cnt_cursor..sketch_cnt_cursor + num_bins] + .to_vec(); + sketch_cnt_cursor += num_bins; + + DecodedPoint::Sketch { + timestamp, + count, + sum_bits: sum.to_bits(), + min_bits: min.to_bits(), + max_bits: max.to_bits(), + bin_keys, + bin_counts, + } + } else { + let value = read_value( + data, + value_type, + &mut sint_cursor, + &mut f32_cursor, + &mut f64_cursor, + ); + DecodedPoint::Value { + timestamp, + bits: value.to_bits(), + } + } + }) + .collect() + }) + .collect() +} + +/// Runs `build` against a [`V3Writer`], and checks that independently decoding the resulting +/// hand-rolled bytes (via [`decode_points`]) reproduces exactly `expected_points` — the values +/// `build` is expected to have passed to `add_point`/`add_sketch`, in the same per-metric, +/// per-point order. Unlike [`assert_wire_parity`], this does not go through `V3Writer` on the +/// "expected" side at all, so it actually exercises delta-encoding and value-type compaction (see +/// the module docs). +#[track_caller] +fn assert_points_round_trip(build: impl Fn(&mut V3Writer), expected_points: &[Vec]) { + let mut writer = V3Writer::new(); + build(&mut writer); + let bytes = writer.finalize().payload; + + let message = pb::MetricData::decode(bytes.as_slice()) + .expect("hand-rolled encoder output must be valid protobuf"); + let decoded = decode_points(&message); + + assert_eq!( + decoded, expected_points, + "decoding the hand-rolled encoder's bytes must reproduce the original point values" + ); +} + +#[test] +fn empty_payload() { + assert_wire_parity(|_writer| {}); +} + +#[test] +fn single_gauge_zero_value() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Gauge, "zero.metric"); + m.add_point(1_000, 0.0).unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Value { + timestamp: 1_000, + bits: 0.0_f64.to_bits(), + }]], + ); +} + +#[test] +fn single_count_small_int_value() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Count, "small.int"); + m.add_point(1_000, 42.0).unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Value { + timestamp: 1_000, + bits: 42.0_f64.to_bits(), + }]], + ); +} + +#[test] +fn single_gauge_large_int_value() { + // Larger than 2^24 but still losslessly representable as sint64. + let value = (1i64 << 40) as f64; + let build = move |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Gauge, "large.int"); + m.add_point(1_000, value).unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Value { + timestamp: 1_000, + bits: value.to_bits(), + }]], + ); +} + +#[test] +fn single_gauge_float32_value() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Gauge, "float32.metric"); + m.add_point(1_000, 1.5).unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Value { + timestamp: 1_000, + bits: 1.5_f64.to_bits(), + }]], + ); +} + +#[test] +fn single_gauge_float64_value() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Gauge, "float64.metric"); + m.add_point(1_000, core::f64::consts::PI).unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Value { + timestamp: 1_000, + bits: core::f64::consts::PI.to_bits(), + }]], + ); +} + +#[test] +fn nan_value_round_trips_as_float64() { + // NaN is a legitimate (if unusual) point value: producers can submit it, and this crate must + // encode it deterministically rather than panicking or silently substituting another value. + // `assert_wire_parity` can't express this case: `MetricData`'s derived `PartialEq` (like IEEE + // 754 itself) treats NaN as unequal to itself, so it would fail even on a correct encoding. + // `assert_points_round_trip` compares bit patterns instead, so it can actually check this. + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Gauge, "nan.metric"); + m.add_point(1_000, f64::NAN).unwrap(); + m.close(); + }; + assert_points_round_trip( + build, + &[vec![DecodedPoint::Value { + timestamp: 1_000, + bits: f64::NAN.to_bits(), + }]], + ); +} + +#[test] +fn mixed_large_int_and_float32_promotes_to_float64() { + // Regression case: a large integer mixed with a fractional float32 value must be stored (and + // therefore wire-encoded) as float64 to avoid precision loss. `large` is deliberately not a + // power of two: 2^30 itself happens to survive an f32 round-trip losslessly (only the + // exponent is used, the mantissa is all zero), so it wouldn't actually detect a regression + // where this promotion is missing and everything gets compacted to float32 instead. 2^30 + 1 + // needs 31 significant bits and does not fit in f32's 24-bit mantissa, so a missing promotion + // would visibly corrupt this value. + let large = ((1i64 << 30) + 1) as f64; + let build = move |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Gauge, "mixed.metric"); + m.add_point(1_000, large).unwrap(); + m.add_point(2_000, 1.5).unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![ + DecodedPoint::Value { + timestamp: 1_000, + bits: large.to_bits(), + }, + DecodedPoint::Value { + timestamp: 2_000, + bits: 1.5_f64.to_bits(), + }, + ]], + ); +} + +#[test] +fn rate_metric_with_interval() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Rate, "rate.metric"); + m.set_interval(60); + m.add_point(1_000, 3.5).unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Value { + timestamp: 1_000, + bits: 3.5_f64.to_bits(), + }]], + ); +} + +#[test] +fn multiple_points_per_metric() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Gauge, "multi.point"); + for i in 0..10 { + m.add_point(1_000 + i * 10, i as f64).unwrap(); + } + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[(0..10) + .map(|i| DecodedPoint::Value { + timestamp: 1_000 + i * 10, + bits: (i as f64).to_bits(), + }) + .collect()], + ); +} + +#[test] +fn multiple_metrics_share_interned_name() { + assert_wire_parity(|writer| { + for i in 0u32..3 { + let mut m = writer.write(V3MetricType::Count, "shared.name"); + m.add_point(1_000 + i64::from(i), f64::from(i)).unwrap(); + m.close(); + } + }); +} + +#[test] +fn tags_are_deduplicated_across_metrics() { + assert_wire_parity(|writer| { + { + let mut m = writer.write(V3MetricType::Gauge, "a"); + m.set_tags(["env:prod", "service:web"].into_iter()); + m.add_point(1_000, 1.0).unwrap(); + m.close(); + } + { + // Overlapping tag (env:prod) plus a new one; exercises both dictionary reuse and a + // brand-new tagset. + let mut m = writer.write(V3MetricType::Gauge, "b"); + m.set_tags(["env:prod", "service:api"].into_iter()); + m.add_point(2_000, 2.0).unwrap(); + m.close(); + } + { + // Exact same tagset as the first metric; exercises tagset (not just tag) dedup. + let mut m = writer.write(V3MetricType::Gauge, "c"); + m.set_tags(["env:prod", "service:web"].into_iter()); + m.add_point(3_000, 3.0).unwrap(); + m.close(); + } + { + let mut m = writer.write(V3MetricType::Gauge, "no.tags"); + m.add_point(4_000, 4.0).unwrap(); + m.close(); + } + }); +} + +#[test] +fn resources_host_and_device_pairs() { + assert_wire_parity(|writer| { + { + let mut m = writer.write(V3MetricType::Gauge, "with.resources"); + m.set_resources(&[("host", "server-1"), ("device", "eth0")]); + m.add_point(1_000, 1.0).unwrap(); + m.close(); + } + { + // Same resource set again: exercises resource-set dedup. + let mut m = writer.write(V3MetricType::Gauge, "same.resources"); + m.set_resources(&[("host", "server-1"), ("device", "eth0")]); + m.add_point(2_000, 2.0).unwrap(); + m.close(); + } + { + let mut m = writer.write(V3MetricType::Gauge, "no.resources"); + m.add_point(3_000, 3.0).unwrap(); + m.close(); + } + }); +} + +#[test] +fn source_type_name_is_encoded() { + assert_wire_parity(|writer| { + let mut m = writer.write(V3MetricType::Count, "with.source.type"); + m.set_source_type("nginx"); + m.add_point(1_000, 1.0).unwrap(); + m.close(); + }); +} + +#[test] +fn origin_metadata_with_no_index_flag() { + assert_wire_parity(|writer| { + let mut m = writer.write(V3MetricType::Gauge, "with.origin"); + m.set_origin(1, 2, 3, true); + m.add_point(1_000, 1.0).unwrap(); + m.close(); + }); +} + +#[test] +fn unit_toggled_on_off_on() { + assert_wire_parity(|writer| { + { + let mut m = writer.write(V3MetricType::Gauge, "has.unit"); + m.set_unit("millisecond"); + m.add_point(1_000, 42.0).unwrap(); + m.close(); + } + { + let mut m = writer.write(V3MetricType::Gauge, "no.unit"); + m.add_point(1_000, 43.0).unwrap(); + m.close(); + } + { + // Reuses the "millisecond" unit dictionary entry. + let mut m = writer.write(V3MetricType::Gauge, "same.unit"); + m.set_unit("millisecond"); + m.add_point(1_000, 44.0).unwrap(); + m.close(); + } + }); +} + +#[test] +fn unit_set_then_cleared() { + assert_wire_parity(|writer| { + let mut m = writer.write(V3MetricType::Gauge, "cleared.unit"); + m.set_unit("byte"); + m.set_unit(""); // clears it back out + m.add_point(1_000, 1.0).unwrap(); + m.close(); + }); +} + +#[test] +fn sketch_with_integer_summary() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Sketch, "sketch.int"); + m.add_sketch( + 1_000, + 5, + 15.0, + 1.0, + 9.0, + &[-2, -1, 0, 1, 2], + &[1, 1, 1, 1, 1], + ) + .unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Sketch { + timestamp: 1_000, + count: 5, + sum_bits: 15.0_f64.to_bits(), + min_bits: 1.0_f64.to_bits(), + max_bits: 9.0_f64.to_bits(), + bin_keys: vec![-2, -1, 0, 1, 2], + bin_counts: vec![1, 1, 1, 1, 1], + }]], + ); +} + +#[test] +fn sketch_with_float_summary() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Sketch, "sketch.float"); + m.add_sketch( + 1_000, + 3, + 4.5, + 0.5, + core::f64::consts::E, + &[-1, 0, 1], + &[2, 3, 1], + ) + .unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![DecodedPoint::Sketch { + timestamp: 1_000, + count: 3, + sum_bits: 4.5_f64.to_bits(), + min_bits: 0.5_f64.to_bits(), + max_bits: core::f64::consts::E.to_bits(), + bin_keys: vec![-1, 0, 1], + bin_counts: vec![2, 3, 1], + }]], + ); +} + +#[test] +fn sketch_with_multiple_points() { + let build = |writer: &mut V3Writer| { + let mut m = writer.write(V3MetricType::Sketch, "sketch.multi"); + m.add_sketch(1_000, 2, 3.0, 1.0, 2.0, &[0, 1], &[1, 1]) + .unwrap(); + m.add_sketch(2_000, 4, 20.0, 1.0, 15.0, &[-3, -2, -1, 0], &[1, 1, 1, 1]) + .unwrap(); + m.close(); + }; + assert_wire_parity(build); + assert_points_round_trip( + build, + &[vec![ + DecodedPoint::Sketch { + timestamp: 1_000, + count: 2, + sum_bits: 3.0_f64.to_bits(), + min_bits: 1.0_f64.to_bits(), + max_bits: 2.0_f64.to_bits(), + bin_keys: vec![0, 1], + bin_counts: vec![1, 1], + }, + DecodedPoint::Sketch { + timestamp: 2_000, + count: 4, + sum_bits: 20.0_f64.to_bits(), + min_bits: 1.0_f64.to_bits(), + max_bits: 15.0_f64.to_bits(), + bin_keys: vec![-3, -2, -1, 0], + bin_counts: vec![1, 1, 1, 1], + }, + ]], + ); +} + +#[test] +fn kitchen_sink_payload() { + // Combines every dimension above into one payload: multiple metric types, shared and + // divergent tag/resource/unit/origin/source-type dictionaries, every value-type compaction + // path, and both point and sketch metrics. + assert_wire_parity(|writer| { + { + let mut m = writer.write(V3MetricType::Count, "requests.count"); + m.set_tags(["env:prod", "service:web", "region:us-east"].into_iter()); + m.set_resources(&[("host", "server-1")]); + m.set_source_type("nginx"); + m.add_point(1_000, 0.0).unwrap(); + m.add_point(1_010, 12.0).unwrap(); + m.close(); + } + { + let mut m = writer.write(V3MetricType::Rate, "requests.rate"); + m.set_tags(["env:prod", "service:web"].into_iter()); + m.set_interval(10); + m.set_unit("request"); + m.add_point(1_000, 1.2).unwrap(); + m.close(); + } + { + let mut m = writer.write(V3MetricType::Gauge, "memory.usage"); + m.set_tags(["env:prod", "service:api"].into_iter()); + m.set_resources(&[("host", "server-1"), ("container", "abc123")]); + m.set_unit("byte"); + m.set_origin(7, 2, 1, false); + m.add_point(1_000, (1i64 << 32) as f64).unwrap(); + m.close(); + } + { + let mut m = writer.write(V3MetricType::Gauge, "cpu.usage"); + m.set_tags(core::iter::once("env:staging")); + m.set_origin(7, 2, 1, true); // reuses origin dict entry, sets no-index flag + m.add_point(1_000, 0.42).unwrap(); + m.add_point(1_010, (1i64 << 30) as f64).unwrap(); // forces float64 alongside a fraction below + m.add_point(1_020, 0.5).unwrap(); + m.close(); + } + { + let mut m = writer.write(V3MetricType::Sketch, "latency.distribution"); + m.set_tags(["env:prod", "service:web"].into_iter()); // reuses an earlier tagset + m.set_unit("millisecond"); + m.add_sketch( + 1_000, + 5, + 25.0, + 1.0, + 12.0, + &[-2, -1, 0, 1, 2], + &[1, 1, 1, 1, 1], + ) + .unwrap(); + m.add_sketch( + 2_000, + 3, + 4.5, + 0.5, + core::f64::consts::E, + &[-1, 0, 1], + &[2, 3, 1], + ) + .unwrap(); + m.close(); + } + { + // No tags, no resources, no unit, no origin, no source type: exercises every "0 = + // empty" dictionary reference path in the same payload as everything above. + let mut m = writer.write(V3MetricType::Gauge, "bare.metric"); + m.add_point(1_000, 99.0).unwrap(); + m.close(); + } + }); +} + +/// Randomized coverage of the writer/encoder's most intricate logic: name and tag interning +/// (dictionary reuse across metrics), delta encoding of the resulting reference columns, and +/// value-type compaction across the zero/int24/int48/float32/float64 boundaries. Each generated +/// case is checked for the same byte-for-byte parity as the examples above. +#[test] +fn wire_bytes_match_prost_for_randomized_metrics() { + use bolero::TypeGenerator as _; + + const NAME_POOL: &[&str] = &["requests", "latency", "errors", "cpu.usage", "memory.usage"]; + const TAG_POOL: &[&str] = &[ + "env:prod", + "env:staging", + "service:web", + "service:api", + "region:us-east", + ]; + + let metric_type_idx = 0u8..=2; // Count, Rate, Gauge (sketches are covered by dedicated tests above) + let name_idx = 0usize..NAME_POOL.len(); + let tag_idx = 0usize..TAG_POOL.len(); + let tags = Vec::::produce().with().values(tag_idx); + let values = Vec::::produce(); + let metrics = Vec::<(u8, usize, Vec, Vec)>::produce() + .with() + .values((metric_type_idx, name_idx, tags, values)); + + bolero::check!() + .with_generator(metrics) + .for_each(|metrics| { + // NaN is scrubbed out up front (and shared between the writer and the expectations + // below): it's encoded deterministically like any other value, but `MetricData`'s + // derived `PartialEq` (like IEEE 754 itself) treats NaN as unequal to itself, which + // would make the equality checks below spuriously fail on a + // correctly-encoded payload. + let metrics: Vec<(u8, usize, Vec, Vec)> = metrics + .iter() + .map(|(type_idx, name_idx, tag_idxs, values)| { + let values = values + .iter() + .map(|v| if v.is_nan() { 0.0 } else { *v }) + .collect(); + (*type_idx, *name_idx, tag_idxs.clone(), values) + }) + .collect(); + + let build = |writer: &mut V3Writer| { + for (type_idx, name_idx, tag_idxs, values) in &metrics { + let metric_type = match type_idx % 3 { + 0 => V3MetricType::Count, + 1 => V3MetricType::Rate, + _ => V3MetricType::Gauge, + }; + let mut m = writer.write(metric_type, NAME_POOL[*name_idx]); + m.set_tags(tag_idxs.iter().map(|&i| TAG_POOL[i])); + m.set_interval(7); + for (i, &value) in values.iter().enumerate() { + // Non-negative, monotonically increasing timestamps (`i` is bounded by the + // generated `Vec`'s length, nowhere near overflowing); the actual values + // are what's under test here. + #[allow(clippy::cast_possible_wrap)] + m.add_point(1_000 + i as i64, value).unwrap(); + } + m.close(); + } + }; + assert_wire_parity(build); + + let expected_points: Vec> = metrics + .iter() + .map(|(_, _, _, values)| { + values + .iter() + .enumerate() + .map(|(i, value)| DecodedPoint::Value { + #[allow(clippy::cast_possible_wrap)] + timestamp: 1_000 + i as i64, + bits: value.to_bits(), + }) + .collect() + }) + .collect(); + assert_points_round_trip(build, &expected_points); + }); +} diff --git a/metrics/dd-metrics-v3/tests/pb/mod.rs b/metrics/dd-metrics-v3/tests/pb/mod.rs new file mode 100644 index 00000000..83b20fa5 --- /dev/null +++ b/metrics/dd-metrics-v3/tests/pb/mod.rs @@ -0,0 +1,215 @@ +// This file is @generated by prost-build from `proto/metrics/intake_v3.proto`. Do not edit it +// directly: regenerate it with `cargo build -p dd-metrics-v3 --features generate-protobuf` after +// changing the proto file. +#![allow(dead_code, clippy::all, clippy::pedantic, clippy::nursery)] + +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Payload { + #[prost(message, optional, tag = "2")] + pub metadata: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub metric_data: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Metadata { + #[prost(string, repeated, tag = "1")] + pub tags: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// even number of elements, \[Type, Name\] pairs + #[prost(string, repeated, tag = "2")] + pub resources: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MetricData { + /// Dictionaries + /// All dictionary indexes are base-1, zero implicitly represents an empty value. + /// + /// varint length + value + #[prost(bytes = "vec", tag = "1")] + pub dict_name_str: ::prost::alloc::vec::Vec, + /// varint length + value + #[prost(bytes = "vec", tag = "2")] + pub dict_tag_str: ::prost::alloc::vec::Vec, + /// length, delta encoded set of indexes into dictTagsStr + #[prost(sint64, repeated, tag = "3")] + pub dict_tagsets: ::prost::alloc::vec::Vec, + /// varint length + value + #[prost(bytes = "vec", tag = "4")] + pub dict_resource_str: ::prost::alloc::vec::Vec, + /// number of elements in Type and Name arrays + #[prost(int64, repeated, tag = "5")] + pub dict_resource_len: ::prost::alloc::vec::Vec, + /// delta encoded set of indexes into dictResourceStr + #[prost(sint64, repeated, tag = "6")] + pub dict_resource_type: ::prost::alloc::vec::Vec, + /// delta encoded set of indexes into dictResourceStr + #[prost(sint64, repeated, tag = "7")] + pub dict_resource_name: ::prost::alloc::vec::Vec, + /// varint length + value + #[prost(bytes = "vec", tag = "8")] + pub dict_source_type_name: ::prost::alloc::vec::Vec, + /// (product, category, service) tuples + #[prost(int32, repeated, tag = "9")] + pub dict_origin_info: ::prost::alloc::vec::Vec, + /// varint length + value + #[prost(bytes = "vec", tag = "25")] + pub dict_unit_str: ::prost::alloc::vec::Vec, + /// One entry per time series + /// + /// type = metricType | valueType | metricFlags + #[prost(uint64, repeated, tag = "10")] + pub types: ::prost::alloc::vec::Vec, + /// index into dictNameStr, entire array is delta encoded + #[prost(sint64, repeated, tag = "11")] + pub name_refs: ::prost::alloc::vec::Vec, + /// index into dictTagsets, entire array is delta encoded + #[prost(sint64, repeated, tag = "12")] + pub tagset_refs: ::prost::alloc::vec::Vec, + /// index into dictResourceLen, entire array is delta encoded + #[prost(sint64, repeated, tag = "13")] + pub resources_refs: ::prost::alloc::vec::Vec, + #[prost(uint64, repeated, tag = "14")] + pub intervals: ::prost::alloc::vec::Vec, + #[prost(uint64, repeated, tag = "15")] + pub num_points: ::prost::alloc::vec::Vec, + /// index into dictSourceTypeName, entire array is delta encoded + #[prost(sint64, repeated, tag = "23")] + pub source_type_name_refs: ::prost::alloc::vec::Vec, + /// index into dictOriginInfo, entire array is delta encoded + #[prost(sint64, repeated, tag = "24")] + pub origin_info_refs: ::prost::alloc::vec::Vec, + /// index into dictUnitStr, value present if flagHasUnit is set, entire array is delta encoded + #[prost(sint64, repeated, tag = "26")] + pub unit_refs: ::prost::alloc::vec::Vec, + /// each metric has numPoints values in this section + /// + /// entire array delta encoded + #[prost(sint64, repeated, tag = "16")] + pub timestamps: ::prost::alloc::vec::Vec, + /// or + #[prost(sint64, repeated, tag = "17")] + pub vals_sint64: ::prost::alloc::vec::Vec, + /// or + #[prost(float, repeated, tag = "18")] + pub vals_float32: ::prost::alloc::vec::Vec, + /// based on valueType + #[prost(double, repeated, tag = "19")] + pub vals_float64: ::prost::alloc::vec::Vec, + #[prost(uint64, repeated, tag = "20")] + pub sketch_num_bins: ::prost::alloc::vec::Vec, + /// per-metric sequence is delta encoded + #[prost(sint32, repeated, tag = "21")] + pub sketch_bin_keys: ::prost::alloc::vec::Vec, + /// sketch summary Sum, Min, Max are encoded as three consecutive elements in one of vals using valueType + /// sketch summary Cnt is always encoded in valInt64 + /// sketch summary Avg is reconstructed as Sum/Cnt in the intake + #[prost(uint32, repeated, tag = "22")] + pub sketch_bin_cnts: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Response { + #[prost(string, tag = "1")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum MetricType { + Unused = 0, + Count = 1, + Rate = 2, + Gauge = 3, + Sketch = 4, +} +impl MetricType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unused => "UNUSED", + Self::Count => "Count", + Self::Rate => "Rate", + Self::Gauge => "Gauge", + Self::Sketch => "Sketch", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNUSED" => Some(Self::Unused), + "Count" => Some(Self::Count), + "Rate" => Some(Self::Rate), + "Gauge" => Some(Self::Gauge), + "Sketch" => Some(Self::Sketch), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ValueType { + /// value is zero, not stored explicitly + Zero = 0, + /// value is stored in valsSint64 + Sint64 = 16, + /// value is stored in valsFloat32 + Float32 = 32, + /// value is stored in valsFloat64 + Float64 = 48, +} +impl ValueType { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Zero => "Zero", + Self::Sint64 => "Sint64", + Self::Float32 => "Float32", + Self::Float64 => "Float64", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "Zero" => Some(Self::Zero), + "Sint64" => Some(Self::Sint64), + "Float32" => Some(Self::Float32), + "Float64" => Some(Self::Float64), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum MetricFlags { + FlagNone = 0, + /// metric should not be indexed (equivalent to origin metric type == agent_hidden in v2) + FlagNoIndex = 256, + /// timeseries has a unit in the unitRefs column + FlagHasUnit = 512, +} +impl MetricFlags { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::FlagNone => "flagNone", + Self::FlagNoIndex => "flagNoIndex", + Self::FlagHasUnit => "flagHasUnit", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "flagNone" => Some(Self::FlagNone), + "flagNoIndex" => Some(Self::FlagNoIndex), + "flagHasUnit" => Some(Self::FlagHasUnit), + _ => None, + } + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000..f2004ae0 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.87.0" +components = ["rustfmt", "clippy"] +profile = "minimal" From 4cd8d1a495033b238a736a74d219b157d5bd68f6 Mon Sep 17 00:00:00 2001 From: Mark Kirichenko Date: Tue, 21 Jul 2026 10:43:09 +0200 Subject: [PATCH 2/2] fix(metrics): canonicalize V3 tag sets before interning The V3 writer interned each metric's tag ids in caller-supplied order but only sorted the emitted dictionary entry. The same logical tag set given in different orders (e.g. `["a","b"]` vs `["b","a"]`) therefore produced distinct `tagsetRefs` and duplicate `dictTagsets` entries instead of deduplicating, inflating payloads for producers with unstable tag ordering. Sort `tag_ids` before interning so the interner key matches the canonical on-wire (sorted) form; drop the now-redundant sort in `encode_tagset`. For a unique tag set the output is byte-identical; reordered duplicates now collapse to one ref and one dictionary entry. Add a unit test covering order-independent dedup, and criterion benchmarks (`benches/tagset.rs`) quantifying the added sort's cost on the cache-hit path (low-single-digit-percent, near-zero for typical tag counts). Signed-off-by: Mark Kirichenko Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mark Kirichenko --- metrics/dd-metrics-v3/Cargo.lock | 476 +++++++++++++++++++++++- metrics/dd-metrics-v3/Cargo.toml | 5 + metrics/dd-metrics-v3/benches/tagset.rs | 109 ++++++ metrics/dd-metrics-v3/src/writer.rs | 43 ++- 4 files changed, 622 insertions(+), 11 deletions(-) create mode 100644 metrics/dd-metrics-v3/benches/tagset.rs diff --git a/metrics/dd-metrics-v3/Cargo.lock b/metrics/dd-metrics-v3/Cargo.lock index ee5746ef..05f45531 100644 --- a/metrics/dd-metrics-v3/Cargo.lock +++ b/metrics/dd-metrics-v3/Cargo.lock @@ -11,12 +11,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "anyhow" version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "bitflags" version = "2.13.1" @@ -85,7 +103,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -116,12 +134,24 @@ dependencies = [ "cc", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.3.0" @@ -138,11 +168,131 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "dd-metrics-v3" version = "0.1.0" dependencies = [ "bolero", + "criterion", "foldhash 0.2.0", "hashbrown 0.16.1", "prost", @@ -201,6 +351,30 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -213,6 +387,17 @@ dependencies = [ "wasip2", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -240,6 +425,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "indexmap" version = "2.14.0" @@ -250,6 +441,26 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -259,6 +470,23 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -295,12 +523,27 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "petgraph" version = "0.8.3" @@ -312,6 +555,40 @@ dependencies = [ "indexmap", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -334,7 +611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.119", ] [[package]] @@ -373,7 +650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -381,7 +658,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.119", "tempfile", ] @@ -392,10 +669,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -460,6 +737,26 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.13.1" @@ -502,12 +799,76 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "syn" version = "2.0.119" @@ -519,6 +880,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -531,6 +903,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "toml_datetime" version = "0.6.3" @@ -554,6 +936,16 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -563,6 +955,70 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -610,5 +1066,11 @@ checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/metrics/dd-metrics-v3/Cargo.toml b/metrics/dd-metrics-v3/Cargo.toml index 355d8ded..f1399f5c 100644 --- a/metrics/dd-metrics-v3/Cargo.toml +++ b/metrics/dd-metrics-v3/Cargo.toml @@ -20,8 +20,13 @@ hashbrown = { version = "0.16", default-features = false } [dev-dependencies] bolero = "0.13" +criterion = "0.5" prost = "0.14" +[[bench]] +name = "tagset" +harness = false + [build-dependencies] prost-build = { version = "0.14", optional = true } diff --git a/metrics/dd-metrics-v3/benches/tagset.rs b/metrics/dd-metrics-v3/benches/tagset.rs new file mode 100644 index 00000000..425ae5c2 --- /dev/null +++ b/metrics/dd-metrics-v3/benches/tagset.rs @@ -0,0 +1,109 @@ +//! Benchmarks for the cost of canonicalizing tag sets before interning. +//! +//! The V3 writer sorts a metric's tag ids before interning them so that the same +//! logical tag set supplied in different orders deduplicates to a single dictionary +//! entry. These benchmarks quantify the cost of that sort on the cache-hit path, +//! where the tag set has already been interned and only the (added) sort is new work. +//! +//! Three groups: +//! - `sort_isolated`: raw `slice::sort_unstable` on `n` shuffled `i64`s — an upper +//! bound on the added per-call work, with no interning noise. +//! - `set_tags_stable`: end-to-end `set_tags` cache hits where the caller always uses +//! the same (insertion) order, so tag ids are already ascending and the sort takes +//! its near-linear fast path. This is the realistic consistent-producer case. +//! - `set_tags_unstable`: end-to-end `set_tags` cache hits where the caller uses a +//! fixed but non-monotonic order, so the sort does genuine work every call. This is +//! the worst realistic per-call overhead for a producer with unstable ordering. + +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; +use dd_metrics_v3::{V3MetricType, V3Writer}; +use std::hint::black_box; + +const SIZES: &[usize] = &[4, 16, 64, 256]; + +/// Builds `n` distinct `key:value` tag strings in insertion order. +fn make_tags(n: usize) -> Vec { + (0..n).map(|i| format!("key{i}:value{i}")).collect() +} + +/// A fixed, deterministic non-monotonic permutation of `0..n` (step 7 is coprime to +/// every `n` in `SIZES`, so this is a genuine permutation). +fn scramble(n: usize) -> Vec { + (0..n).map(|k| (k * 7 + 3) % n).collect() +} + +/// Raw cost of the added `sort_unstable`, isolated from interning. +fn bench_sort_isolated(c: &mut Criterion) { + let mut group = c.benchmark_group("sort_isolated"); + for &n in SIZES { + // Shuffled ids so the sort cannot take a sorted/reverse-sorted fast path. + let scrambled: Vec = scramble(n).into_iter().map(|i| i as i64 + 1).collect(); + group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { + b.iter_batched_ref( + || scrambled.clone(), + |v| v.sort_unstable(), + BatchSize::SmallInput, + ); + }); + } + group.finish(); +} + +/// End-to-end `set_tags` on the cache-hit path with consistent (insertion) ordering. +/// Ids are already ascending, so the sort hits its near-linear fast path. +fn bench_set_tags_stable(c: &mut Criterion) { + let mut group = c.benchmark_group("set_tags_stable"); + for &n in SIZES { + let tags = make_tags(n); + group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { + b.iter_batched_ref( + || seeded_writer(&tags, &(0..n).collect::>()), + |w| write_metric(w, &tags, &(0..n).collect::>()), + BatchSize::SmallInput, + ); + }); + } + group.finish(); +} + +/// End-to-end `set_tags` on the cache-hit path with a fixed non-monotonic ordering, so +/// the sort does genuine work on every call. Worst realistic per-call overhead. +fn bench_set_tags_unstable(c: &mut Criterion) { + let mut group = c.benchmark_group("set_tags_unstable"); + for &n in SIZES { + let tags = make_tags(n); + let order = scramble(n); + group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { + b.iter_batched_ref( + || seeded_writer(&tags, &order), + |w| write_metric(w, &tags, &order), + BatchSize::SmallInput, + ); + }); + } + group.finish(); +} + +/// Creates a writer with `tags` (presented in `order`) already interned, so subsequent +/// `set_tags` calls with the same tag set take the cache-hit path. +fn seeded_writer(tags: &[String], order: &[usize]) -> V3Writer { + let mut w = V3Writer::new(); + write_metric(&mut w, tags, order); + w +} + +/// Writes a single count metric whose tags are `tags` presented in `order`. +fn write_metric(w: &mut V3Writer, tags: &[String], order: &[usize]) { + let mut m = w.write(V3MetricType::Count, "bench.metric"); + m.set_tags(black_box(order.iter().map(|&i| &tags[i]))); + m.add_point(1000, 1.0).unwrap(); + m.close(); +} + +criterion_group!( + benches, + bench_sort_isolated, + bench_set_tags_stable, + bench_set_tags_unstable +); +criterion_main!(benches); diff --git a/metrics/dd-metrics-v3/src/writer.rs b/metrics/dd-metrics-v3/src/writer.rs index dd3354fd..c10f87e4 100644 --- a/metrics/dd-metrics-v3/src/writer.rs +++ b/metrics/dd-metrics-v3/src/writer.rs @@ -574,6 +574,13 @@ impl V3Writer { return 0; } + // Canonicalize by sorting the tag ids before interning. A tag set is an + // unordered collection, and its on-wire dictionary form is sorted, so callers + // that supply the same tags in different orders must resolve to the same ref + // and a single dictionary entry. Sorting here (rather than only in + // `encode_tagset`) makes the interner key match that canonical form. + self.tag_ids.sort_unstable(); + let (id, is_new) = self.tagset_interner.get_or_insert(&self.tag_ids); if is_new { self.encode_tagset(); @@ -588,11 +595,9 @@ impl V3Writer { let start = self.dict_tagsets.len(); - // Add all tag IDs + // `tag_ids` is already sorted by `intern_tagset` (its canonical form), so we + // only need to delta-encode here. self.dict_tagsets.extend_from_slice(&self.tag_ids); - - // Sort and delta-encode the tagset portion - self.dict_tagsets[start..].sort_unstable(); delta_encode(&mut self.dict_tagsets[start..]); } @@ -1485,4 +1490,34 @@ mod tests { }) ); } + + #[test] + fn test_tagset_dedup_is_order_independent() { + let mut writer = V3Writer::new(); + + { + let mut m1 = writer.write(V3MetricType::Count, "metric1"); + m1.set_tags(["a", "b"].iter()); + m1.add_point(1000, 1.0).unwrap(); + m1.close(); + } + + { + let mut m2 = writer.write(V3MetricType::Count, "metric2"); + m2.set_tags(["b", "a"].iter()); + m2.add_point(1000, 1.0).unwrap(); + m2.close(); + } + + let data = writer.into_columns(); + + // Both metrics carry the same logical tag set, so they must resolve to the + // same tagset ref. `tags` is delta-encoded, so the second entry's delta must + // be zero. + assert_eq!(data.tags, alloc::vec![1, 0]); + + // The dictionary must contain exactly one tag set entry: a length prefix of 2 + // followed by two delta-encoded tag ids. + assert_eq!(data.dict_tagsets, alloc::vec![2, 1, 1]); + } }