diff --git a/Cargo.lock b/Cargo.lock index 7bc5122..09d7d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -923,6 +923,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hybrid-array" version = "0.4.13" @@ -947,6 +953,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -2918,6 +2925,26 @@ dependencies = [ "tokio-postgres", ] +[[package]] +name = "walshadow-peerdb" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "clap", + "http-body-util", + "hyper", + "hyper-util", + "serde", + "serde_json", + "serde_urlencoded", + "tempfile", + "tokio", + "toml", + "tracing", + "tracing-subscriber", +] + [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 8532e59..f6f0850 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["clickhouse-c-rs", "bench"] +members = ["clickhouse-c-rs", "bench", "walshadow-peerdb"] [package] name = "walshadow" @@ -45,7 +45,7 @@ async-trait = "0.1" backon = { version = "1", features = ["tokio-sleep"] } bytes = "1" chrono = { version = "0.4", default-features = false, features = ["clock"] } -clap = { version = "4", features = ["derive"] } +clap = { version = "4", features = ["derive", "env"] } crc32c = "0.6" fallible-iterator = "0.2" futures = "0.3" @@ -72,3 +72,4 @@ mimalloc = "0.1.52" [dev-dependencies] tempfile = "3" +toml = { version = "1", default-features = false, features = ["parse", "display", "serde"] } diff --git a/docker/Dockerfile b/docker/Dockerfile index 10e6f69..d000504 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,6 +22,7 @@ WORKDIR /src COPY Cargo.toml Cargo.lock ./ COPY clickhouse-c-rs ./clickhouse-c-rs COPY bench ./bench +COPY walshadow-peerdb ./walshadow-peerdb COPY src ./src # Cache mounts persist cargo's registry/git and the target dir across builds, # so a source change recompiles only the changed crates, not all deps. The @@ -63,7 +64,10 @@ RUN chmod +x /usr/local/bin/walshadow-entrypoint \ /var/lib/walshadow/shadow-data \ /var/lib/walshadow/out \ /var/lib/walshadow/spill \ - /var/run/postgresql + /var/run/postgresql \ + /var/run/walshadow \ + /etc/walshadow \ + /etc/walshadow/ch-config.d USER postgres ENTRYPOINT ["/usr/local/bin/walshadow-entrypoint"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 1714d6d..29b82f9 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -15,6 +15,10 @@ mkdir -p "$SHADOW_DATA" chmod 700 "$SHADOW_DATA" mkdir -p "$OUT_DIR" "$SPILL_DIR" "$SOCKET_DIR" +# conf.d drop-in dir the control API writes fragments into (base is ro). +CH_CONFIG="${WALSHADOW_CH_CONFIG:-/etc/walshadow/ch-config.toml}" +mkdir -p "${CH_CONFIG%.toml}.d" + # Pool sizes default here for the local stack, but the EC2 deploy.sh forwards # explicit --decoder-pool-size/--inserter-pool-size via "$@"; clap rejects a # flag passed twice, so only inject our defaults when the caller didn't. @@ -49,6 +53,7 @@ exec walshadow-stream \ --walsender-bind 127.0.0.1:5433 \ --ch-config "${WALSHADOW_CH_CONFIG:-/etc/walshadow/ch-config.toml}" \ --metrics-bind 0.0.0.0:9484 \ + --control-socket "${WALSHADOW_CONTROL_SOCKET:-/var/run/walshadow/control.sock}" \ --status-interval "${WALSHADOW_STATUS_INTERVAL:-5}" \ "${POOL_ARGS[@]}" \ "$@" diff --git a/plans/INDEX.md b/plans/INDEX.md index 8ced945..2d0fff7 100644 --- a/plans/INDEX.md +++ b/plans/INDEX.md @@ -35,6 +35,11 @@ Cross-doc terminology is collected in [GLOSSARY.md](GLOSSARY.md) shared insert tail - [ops.md](ops.md) — preflight, metrics, retention, manifest (floor, 6 LSNs), standby-status triple, kill-restart drill +- [control.md](control.md) — in-process control plane: `ctl` unix-socket + line protocol, base+`conf.d` config merge (API writes only its + `50-api.toml` fragment), live reload (mappings/budgets/CH-conn/table + selection/pause) with no restart, config-driven table opt-in, pause as + `[stream] paused`, `Reloader` (no session lifecycle) - [oracle.md](oracle.md) — differential decode oracle, walshadow PG extension, `--validate` sampling - [clickhouse-c-rs Safety model](../clickhouse-c-rs/README.md#safety-model) diff --git a/plans/config.md b/plans/config.md index 70e660f..7830167 100644 --- a/plans/config.md +++ b/plans/config.md @@ -255,10 +255,13 @@ point, not the consumer set. Four consumers: ## SIGHUP -`spawn_sighup_handler` holds the resolver and calls `reload()` on each signal. No -resolver (metrics-only run, no `--ch-config`) makes it a no-op tap. Install -failure drops the resolver, so `has_changed` returns `Err` and subscribers freeze -at the boot snapshot — reload disabled, config still serves. +`run` installs the handler at boot; install failure is fatal, so a running +daemon always has the reload path armed. `spawn_sighup_reload` holds the +process-lifetime `Reloader` and calls `Reloader::reload()` on each signal — +the same path as the control socket's `config reload` +([control.md](control.md)). The `Reloader` carries the running session's +resolver (`set_resolver`); with none registered (metrics-only run, no +`--ch-config`) reload is a no-op tap. ## Mapping lifecycle diff --git a/plans/control.md b/plans/control.md new file mode 100644 index 0000000..44dc25d --- /dev/null +++ b/plans/control.md @@ -0,0 +1,175 @@ +# control — in-process control plane + live reconfigure + +The daemon (`walshadow-stream`) embeds a control plane: a request/response Unix +socket for management, config as a merged TOML with a `conf.d` drop-in, and +**live reload** so source/dest/table/pause changes apply without dropping +connections or restarting. No separate binary, no child process. The intended +external consumer is the `walshadow-peerdb` HTTP shim (branch `origin/peerdbapi`, +`walshadow-peerdb/`), which translates PeerDB's flow API onto this socket. + +## Control socket + `ctl` + +`--control-socket ` binds a `UnixListener` (`src/ops/control.rs::serve`, +modeled on `metrics::serve` + the `shadow_stream.rs` bind pattern). Absent → +disabled. The client is the same binary: `walshadow-stream ctl ` with the +command body as a TOML fragment on stdin (`ctl apply <` header +line selects the handler; everything after the first newline is the config, a +TOML document, read to EOF (the client half-closes its write side). TOML keeps +full value typing (int/float/±inf/nan/array) and quotes arbitrary strings, so a +value may contain spaces, `=`, newlines. Response: `OK\n` + an optional payload +that is *also* TOML (`show`/`status` are tables, `tables`/`columns` are +`[[tables]]`/`[[columns]]` arrays, `schemas` an array of strings), or +`ERR \n`. `Request::parse` + `encode_request` are the codec. + +### Verbs +- `apply` — deep-merge the request body (any sections) into the fragment, + validate the merged effective config the way boot does + (`EmitterConfig::from_table`), reload. A fragment that won't parse is rejected + and rolled back so it can't wedge the next reload / restart. Connection config, + table opt-in (`[table..] replicate = true`, `initial_load`), and pause + (`[stream] paused = true`) are all just sections in the body. +- `unset` — mask keys out of the fragment, then validate + reload. The body + mirrors the config shape (same TOML dialect as `apply`, inverted): a non-table + value — the sentinel `""` — removes that key including any subtree, a section + recurses. So `[source] password = ""` drops one key, `[table] demo = ""` one + namespace, top-level `table = ""` the whole section. The sole delete + primitive; only the API's own fragment is touched. +- `reload` — live reload (re-read merged config + republish; SIGHUP over the + socket). +- `show` — merged effective config (passwords masked). +- `status` — a TOML table: `state = "running"|"paused"` + `rows_synced`, + `backfills_pending`, `lag_bytes`, `lag_seconds`, `uptime_secs` (all from the + metrics snapshot). +- `tables` (`namespace`) — enumerate source `pg_class` as an `[[tables]]` array + (`namespace`, `name`, `selected`, `replica_identity`), marking `selected` from + the merged config; `schemas` (array of strings), `columns` (`namespace`, + `relname` → `[[columns]]` with `name`, `type`, `notnull`) — source-PG + introspection. + +`SharedCtx { ch_config, source_base, metrics, reloader, frag_lock }` is handed to +the handlers; `frag_lock` serializes fragment read-modify-write so concurrent +`apply`/`unset` can't lose an update or race a rollback. `Reloader` holds only +the running session's `Arc` (`set_resolver`, `reload`) — there is +no start/stop/restart state machine. + +## Config: base + conf.d merge + +Config is the daemon's own TOML, `--ch-config` **plus** every `*.toml` in the +sibling `.d/` directory (e.g. `ch-config.toml` → `ch-config.d/`), +deep-merged in lexical filename order — Postgres `include_dir` style +(`ch_emitter::load_merged` / `merge_tables`). `load_effective(path, base)` layers +the CLI-arg `[source]` defaults *under* the file so source connection resolves +file-over-CLI (matches `EmitterConfig` boot). + +The control API writes **only its own fragment**, `ch-config.d/50-api.toml` +(`frag_path`) — sparse, only the keys `apply`/`unset` set. The operator's base +`ch-config.toml` is never rewritten (can be read-only mounted); other channels +can own other fragments. `show`/introspection/`status` read the merged +effective config (`get_config` = `load_effective`). + +Sections: `[source]` (source conn), `[ch]` (dest conn + emitter knobs), +`[table..]`, `[namespace.]`, `[runtime_config] schema`, +`[stream] paused`. `EmitterConfig::from_table` parses a merged table (the thin +`from_toml_str` wrapper parses a string then calls it). + +## Live reconfigure (no restart) + +SIGHUP (`spawn_sighup_reload`, unconditional — independent of the control +socket) and `ctl reload` both call `Reloader::reload` → `ConfigResolver::reload` +→ re-read via `load_effective`, rebuild `inner.base`, `republish` on the watch. +Consumers pick it up live: + +- **mappings / namespaces / budgets / compression / drop-strategy / retry** — + batcher + inserter + mapping-refresher + DDL applicator already read the watch + (`src/config.rs`). +- **ClickHouse connection** — `ResolvedConfig` carries the CH conn fields; the + inserter pool (`pipeline/inserter.rs`) and `DdlApplicator` (`ch_ddl.rs`) + compare the conn tuple off the watch and `reconnect()` at a batch/apply + boundary (same mechanism as compression). +- **table selection** — `ResolvedConfig.table_opt_ins` carries the columns-less + `[table.*]` opt-in intents; the reorder coordinator + (`pipeline/reorder.rs::maybe_apply_reload`, called per commit in the drain + driver) diffs desired-vs-applied and runs `apply_table_opt_in` (add, + auto-create) / `exclude_table` + `note_opt_out` (remove, CH table retained). + Applies at the next commit (deferred while idle). +- **pause** — `[stream] paused` in `ResolvedConfig`; the pump reads it live from + a `config_rx` and gates the `feed.next_chunk` `select!` arm off when paused. + +**Source connection is not live**: `reload()` doesn't touch the pump's +`SourceFeed`. A DNS change is picked up by the pump's existing +`reconnect_or_fatal`; a real host change needs a process restart. + +## Table selection is config-driven + +`[table..]` with `columns` → pinned mapping (`EmitterConfig.tables`, as +before). Without `columns` → an opt-in intent (`runtime_config::TableRow` into +`EmitterConfig.table_opt_ins`), materialized via `apply_table_opt_in` +(descriptor-derived auto-create, optional `initial_load` backfill) — the exact +path the source-PG `config_table` overlay uses. So opting a table in is +`ctl apply <<'[table..]' replicate = true` (out → `replicate = false`, +or `unset` the block), applied on reload — **no source-PG writes from control**. +Because `apply` is a deep-merge, opting one table in leaves every other opt-in +and every operator-pinned base mapping alone. The `config_table` + WAL overlay +([config.md], [future/runtime_config_from_pg.md]) still exists independently for +direct-PG operators. + +## Pause + +`apply` of `[stream] paused = true` writes the flag to the fragment + reloads; +the pump stops consuming source WAL (idles at `next_chunk`), freezing its LSN and +the slot's confirmed position — nothing downstream drops. `paused = false` +resumes; the pump continues from the same LSN, so every table picks up where it +left off. +Retention across a pause requires a replication slot (`[source] slot`); without +one, `wal_keep_size` bounds pause duration before the source recycles WAL. A +pause longer than `wal_sender_timeout` may drop the replication connection; +resume reconnects from the slot (still no data loss with a slot). + +## Lifecycle + +The daemon runs **one** streaming session (`run_session`), forever, until Ctrl-C +/ CopyDone / fatal. `run` binds metrics + control socket + SIGHUP, then calls +`run_session` once; Ctrl-C is a pump-loop `select!` arm that breaks and drains +the pipeline gracefully. There is no supervisor loop, no restart, no +running/stopped/exited machine — reconfigure is always a live reload; pause is a +config flag. + +## Deploy + +`docker/entrypoint.sh` passes `--control-socket` and `mkdir -p +"${CH_CONFIG%.toml}.d"` so the API can drop fragments; the image creates +`/etc/walshadow/ch-config.d` (postgres-owned). Base `ch-config.toml` stays a +read-only mount. + +## Files +- `src/ops/control.rs` — socket, protocol, handlers, `Reloader`, `SharedCtx`. +- `src/bin/stream.rs` — `--control-socket`, `ctl` subcommand, `run`/`run_session`, + `cli_source_base`, `spawn_sighup_reload`, pump `paused` gate. +- `src/emit/ch_emitter.rs` — `load_merged`/`load_effective`/`merge_tables`, + `from_table` (columns-optional), `EmitterConfig.{table_opt_ins,paused}`. +- `src/config.rs` — `ResolvedConfig` (CH conn + `table_opt_ins` + `paused`), + `reload` via `load_effective`, `ConfigResolver.cli_source_base`. +- `src/emit/pipeline/inserter.rs`, `src/emit/ch_ddl.rs` — live CH reconnect. +- `src/emit/pipeline/reorder.rs` — `maybe_apply_reload` opt-in diff at commit. + +## Status / open edges +- e2e-verified live: pause/resume + table add (auto-create) via `ctl apply` + (`[stream] paused` / `[table.*] replicate`) + `ctl reload` and via SIGHUP, no + restart, base file untouched. +- Live CH-connection swap needs a second ClickHouse to test fully. +- `ToastResolver` live CH reconnect (for `[toast] mode = disk`) is a TODO. +- Control still opens source-PG read connections for introspection + (`// TODO` on `pg_connect`) — route through the daemon's catalog later. +- The `walshadow-peerdb` shim's PAUSED/RUNNING map to `apply [stream] paused`; + create-mirror maps to one `apply` carrying `[source]` + `[ch]` + `[table.*]` + (source, dest, and tables in a single atomic reload). diff --git a/plans/future/INDEX.md b/plans/future/INDEX.md index 36ee720..3131629 100644 --- a/plans/future/INDEX.md +++ b/plans/future/INDEX.md @@ -16,6 +16,7 @@ surface; promote into `plans/` once built * [coverage100.md](coverage100.md) — drive `cargo llvm-cov` line coverage toward 100%: tiered work list (pure units → fixtures → live e2e → hard tail) * [FUZZ.md](FUZZ.md) — continuous coverage-guided fuzzing (cargo-fuzz/libFuzzer) across wal-rus + walshadow + clickhouse-c-rs: tiered targets, round-trip/differential oracles, C-boundary ASan, unattended-VM supervisor * [pipeline_backpressure_and_scaling.md](pipeline_backpressure_and_scaling.md) — parallel decode+insert pipeline: WAL-pump backpressure via wire/record split, decode/insert scaling (bootstrap Option B, hot-table sharding, N/M sizing); pipeline substrate in [emitter.md](../emitter.md) +* [peerdb.md](peerdb.md) — `walshadow-peerdb/` crate: PeerDB flow HTTP API shim translating onto the control daemon's unix-socket protocol; endpoint map, accept-&-ignore surface, control-protocol extensions * [dependencies.md](dependencies.md) — crates.io replacement candidates for generic object storage, MPMC, retry, throttling, and metrics code * [risks.md](risks.md) — measurement-deferred risks and open questions * [parked.md](parked.md) — small operational polish + cross-major fixtures + skipped-test drive diff --git a/plans/future/peerdb.md b/plans/future/peerdb.md new file mode 100644 index 0000000..0e413fa --- /dev/null +++ b/plans/future/peerdb.md @@ -0,0 +1,212 @@ +# peerdb API shim (`walshadow-peerdb/`) + +Second workspace crate exposing PeerDB's flow HTTP API — the grpc-gateway JSON +surface over `FlowService` (PeerDB `protos/route.proto`) — as a thin translator +onto the control daemon's TOML socket protocol (`ops/control.rs`). Goal: control +planes and UIs that already speak PeerDB (ClickPipes, peerdb-ui) drive walshadow +unchanged. The shim owns zero replication logic and none of walshadow's WAL / +native-protocol dependencies; process supervision, config persistence, streamer +launch stay in the control daemon. Dependency surface: tokio, serde, `toml`, an +HTTP server (bare hyper), unix-socket client + +``` +PeerDB client ──HTTP/JSON──▶ walshadow-peerdb ──unix socket──▶ walshadow-control ──▶ walshadow-stream +``` + +## Topology & cardinality + +One shim ↔ one control socket ↔ one streamer. Mirror cardinality is exactly +one: `CreateCDCFlow` records `flow_job_name` and echoes it as `workflow_id`; +a second create without `attach_to_existing` returns `ALREADY_EXISTS`; with it, +returns the recorded id. `ListMirrors` / `ListMirrorNames` return a singleton +(or empty) list. Multi-pipe deployments run N containers, each with its own +control daemon + shim — no in-shim scheduling. Default bind `:8113`, matching +PeerDB's HTTP gateway port so existing client config carries over + +## Wire fidelity + +- **proto3-JSON per grpc-gateway**: lowerCamelCase fields, enums as strings + (`"STATUS_RUNNING"`), 64-bit ints as strings, absent field = default value. + Deserialization is tolerant: unknown fields ignored, missing fields + defaulted — matches proto3 semantics, so PeerDB clients evolve without + lockstep shim releases +- **Hand-written serde structs** for the consumed subset, not prost/pbjson + codegen from vendored protos. Full codegen drags in hundreds of messages for + a surface that is mostly stubs; the consumed subset is ~15 messages. Revisit + if field drift becomes a recurring bug source +- **Errors** in grpc-gateway shape `{"code": , "message": …}` with + the gateway's HTTP status mapping (3→400, 5→404, 6→409, 12→501, 13→500, + 14→503). `ERR ` from the control socket maps to code 13 unless the + handler knows better; socket connect failure maps to 14 +- **Auth**: honor the `Authorization` header against a `PEERDB_PASSWORD`-style + env var (constant-time compare), unauthenticated when unset — mirrors + PeerDB gateway behavior + +## Endpoint map + +Four classes. *Mapped* endpoints drive the control socket; *served* endpoints +answer from shim/control state without side effects; *accept & ignore* return +success-shaped empty bodies so callers proceed; *reject* returns +`UNIMPLEMENTED` + +### Mapped + +| route | control action | +|---|---| +| `POST /v1/peers/create` | `postgres_config` → `apply` `[source]`; `clickhouse_config` → `apply` `[ch]`; peer name recorded in registry | +| `POST /v1/peers/validate` | structural check only — the protocol has no non-persisting connection probe (see Validation below) | +| `POST /v1/peers/drop` | forget registry entry; refuse while the mirror references it | +| `POST /v1/mirrors/cdc/validate` | `tables` — connecting to the applied source proves reachability + table existence in one call; no destination probe | +| `POST /v1/flows/cdc/create` | resolve `sourceName`/`destinationName` against registry, then one `apply` carrying `[table..] replicate = true` per mapping plus `[stream] paused = false`; `workflow_id` = `flow_job_name` | +| `POST /v1/mirrors/state_change` | `STATUS_PAUSED` → `apply [stream] paused = true`; `STATUS_RUNNING` → `apply [stream] paused = false`; `STATUS_TERMINATED` → pause + `unset table` + forget mirror; `flowConfigUpdate.additionalTables`/`removedTables` → `apply` the opted-in blocks, `unset` the dropped ones | +| `POST /v1/mirrors/status` | `status` → `FlowStatus` (mapping below) + `CDCMirrorStatus` skeleton | +| `GET /v1/mirrors/list`, `/v1/mirrors/names` | singleton from mirror record + live `status` | + +`TableMapping.sourceTableIdentifier` splits into (namespace, relname) at +ingress; dotted strings exist only at control-line interpolation. +`destinationTableIdentifier` differing from source naming is rejected until +per-table target rename exists in runtime config +([runtime_config_from_pg.md](runtime_config_from_pg.md)) + +### Served from state / introspection + +| route | source | +|---|---| +| `GET /v1/peers/list`, `/info/{name}`, `/type/{name}` | registry; `peerdb_redacted` fields masked | +| `GET /v1/peers/schemas`, `/tables`, `/tables/all`, `/columns` | source-PG introspection via the `schemas` / `tables` (optional `namespace`) / `columns` (`namespace` + `relname`) verbs, which connect to the applied source | +| `GET /v1/peers/slots/{peer}`, `/stats/{peer}` | synthesized from `status` lag metrics; physical slot presented in logical-slot clothing, `active` = not paused | +| `GET /v1/mirrors/cdc/batches/*`, `cdc/graph`, `cdc/table_total_counts`, `total_rows_synced` | synthesized from metrics scrape (`emitter_rows` etc); one coarse synthetic batch per response, enough for UI rendering | +| `GET /v1/peers/columns/all_type_conversions` | static table of walshadow's PG→CH type map | +| `GET /v1/version`, `/v1/instance/info` | shim + streamer version, ready flag | + +### Accept & ignore + +`GET /v1/peers/publications` (empty list — walshadow consumes physical WAL, +publications don't exist in the model), alerts config CRUD, scripts CRUD, +dynamic settings, flow tags, maintenance + status + skip-snapshot-wait, +`sequences/reset`, `cancel_table_addition`, `slots/lag_history` (empty +series). Within accepted requests, ignored fields: `publicationName`, +`replicationSlotName`, `softDeleteColName`, `syncedAtColName`, snapshot +partition/parallelism knobs, `env`, `script`, `system`. Ignored non-empty +fields log at WARN once per key so silent divergence is greppable + +### Reject (`UNIMPLEMENTED`) + +`POST /v1/flows/qrep/create` (no qrep engine), `initialSnapshotOnly`, +`resync`. Faking success here would make callers believe a load ran + +## FlowStatus mapping + +| control `status` | FlowStatus | +|---|---| +| not paused, `backfills_pending > 0` | `STATUS_SNAPSHOT` | +| not paused | `STATUS_RUNNING` | +| `paused = true` | `STATUS_PAUSED` | +| socket unreachable | `STATUS_UNKNOWN` | +| mirror forgotten | `STATUS_TERMINATED` | + +`paused` reflects the `[stream] paused` config flag, not streamer liveness; a +live daemon always answers running or paused, so UNKNOWN means the socket did +not answer (list/instance endpoints degrade to it, others surface 503). +`STATUS_PAUSING`/`STATUS_TERMINATING` transients unused — an `apply` reloads +synchronously + +## Shim state + +PeerDB persists peers/mirrors in a catalog PG; the shim persists a small JSON +state file (peer name → role + submitted config, mirror record: name, +table mappings, created-at). Connection-parameter truth stays in the control +daemon's state; the shim's copy exists to echo `GetPeerInfo` and to re-derive +`source`/`dest` role on peer reference. Single writer (the shim), same +durability model as the control daemon's `state.json` + +## Control protocol + +The daemon speaks a TOML socket protocol (`ops/control.rs`): one +`\n` request per connection, EOF-framed, answered `OK\n[toml]` +or `ERR `. TOML bodies preserve scalar types and carry values with spaces +(passwords), so the shim needs no client-side quoting. Verbs it drives: + +- `apply` / `unset` — merge a TOML fragment into `ch-config.d/50-api.toml`, or + mask keys out of it; each validates the merged config and live-reloads, and + only ever touches that one fragment so operator-owned base config stays + read-only +- `status` — `paused`, `rows_synced`, `backfills_pending`, `lag_bytes`, + `lag_seconds`, `uptime_secs` +- `tables` (optional `namespace`) / `schemas` / `columns` (`namespace` + + `relname`) — source-PG introspection; `[[tables]]` carry `selected` and + `replica_identity`, `[[columns]]` carry `name` / `type` / `notnull` + +### Validation + +The protocol has no non-persisting connection probe — `apply` mutates and +reloads. So `ValidatePeer` is structural (supported type + host present); +connectivity surfaces when `create_peer` applies the config, or on +`mirrors/cdc/validate`, which lists source tables over the live socket. There +is no destination probe: ClickHouse reachability first shows when the stream +runs. Per-column key membership isn't in the `columns` reply, so introspected +columns report `isKey`/`isReplicaIdentity` false + +## Anti-goals + +- **No Temporal semantics.** `workflow_id` is an echo of `flow_job_name`; no + workflow history, retries, or signals +- **No publication / logical-slot management.** Physical WAL consumption; + publication fields accepted and dropped +- **No qrep engine.** CDC only +- **No multi-mirror scheduling.** Cardinality one per daemon; N pipes = N + deployments +- **No catalog metadata schema on source.** Nothing like `_peerdb_internal`; + shim state is a local file +- **No soft-delete / synced-at column emulation.** Destination shape is + walshadow's `_lsn` ReplacingMergeTree convergence model, not + `_peerdb_is_deleted` / `_peerdb_synced_at`; readers of destination tables + see walshadow's schema + +## Open questions + +- **gRPC listener.** Consumers wired to the flow API's gRPC port (8112) + rather than the HTTP gateway get nothing from an HTTP-only shim. A tonic + front sharing the handler layer is additive later; confirm what the target + control plane actually speaks before building it +- **Destination-table lifecycle on terminate.** PeerDB drops destination + tables unless `skipDestinationDrop`; control never drops them, so terminate + behaves as `skipDestinationDrop = true` always. Visible to callers that + recreate mirrors expecting a clean destination +- **Initial load fidelity.** `doInitialSnapshot` maps onto walshadow + `initial_load` backfill; partitioning/parallelism knobs have no + counterpart. `InitialLoadSummary` needs per-table backfill progress + surfaced through control status before it can answer honestly +- **TableMapping column controls.** `exclude`, per-column settings, `engine`, + `partitionByExpr` map naturally onto runtime-config column/table overrides — + several of which are themselves future work + ([runtime_config_from_pg.md](runtime_config_from_pg.md)). Until then: + accept-and-ignore with WARN, or reject non-empty? Rejection is honest but + may block UI-driven creates that always send defaults +- **Peer names vs stored config drift.** Registry keeps the submitted peer + config; control keeps the applied one. An operator editing via the control + CLI directly leaves the shim's echo stale. Option: `GetPeerInfo` re-reads + the daemon's `show` config and merges, treating control as truth for + connection fields +- **Type-conversion endpoint fidelity.** UI column-type pickers read + `all_type_conversions`; serving walshadow's real map constrains what the UI + offers. Serving empty disables pickers — probably the safer start + +## Acceptance drills + +- **curl lifecycle.** Create PG peer, create CH peer, validate both, create + CDC mirror over two tables → `status` not paused, `MirrorStatus` = + `STATUS_RUNNING`. Insert rows on source → `total_rows_synced` climbs. + `state_change` PAUSED pauses the streamer; RUNNING resumes; + `flowConfigUpdate.additionalTables` grows the opt-in set and backfills; + TERMINATED stops + clears, `mirrors/list` empties +- **peerdb-ui smoke.** UI pointed at the shim renders peer list, mirror + overview, and status page without errors — batches/graph endpoints return + well-formed empties, redacted peer info displays +- **Ignore surface.** `POST /v1/alerts/config` returns success shape; + `GET /v1/peers/publications` returns empty list; qrep create returns 501 + with grpc-shaped body; create request carrying `softDeleteColName` succeeds + and logs one WARN +- **Tolerant decode.** Create requests from a PeerDB release newer than the + shim (extra unknown fields) parse and apply; absent optional fields behave + as proto3 defaults diff --git a/src/bin/stream.rs b/src/bin/stream.rs index 8abe0cf..4296aad 100644 --- a/src/bin/stream.rs +++ b/src/bin/stream.rs @@ -175,6 +175,25 @@ impl RecordSink for DaemonSinks { } } +/// `walshadow-stream ctl `: drive a running daemon's control socket. +/// Detected before daemon-arg parsing so `ctl` needn't supply daemon args. +#[derive(Debug, Parser)] +#[command( + name = "walshadow-stream ctl", + about = "Control a running walshadow-stream daemon." +)] +struct CtlArgs { + #[arg( + long, + env = "WALSHADOW_CONTROL_SOCKET", + default_value = "/run/walshadow/control.sock" + )] + socket: PathBuf, + /// Control verb, such as `status` or `apply`, read TOML body from stdin + #[arg(trailing_var_arg = true, required = true)] + request: Vec, +} + #[derive(Debug, Parser)] #[command( name = "walshadow-stream", @@ -322,6 +341,9 @@ struct Args { /// HTTP/Prometheus metrics bind address. Disabled when absent. #[arg(long)] metrics_bind: Option, + /// Control socket path, omit to disable control API + #[arg(long)] + control_socket: Option, /// OTLP/gRPC endpoint for traces, e.g. `http://localhost:4317`. Absent /// disables tracing (zero overhead); falls back to /// `OTEL_EXPORTER_OTLP_ENDPOINT`. Spans emit at the `walshadow::trace` @@ -383,6 +405,14 @@ struct Args { #[tokio::main(flavor = "multi_thread", worker_threads = 4)] async fn main() -> Result<()> { + // `ctl` client mode is detected before daemon-arg parsing so it needn't + // supply the daemon's required args. + let argv: Vec = std::env::args().collect(); + if argv.get(1).map(String::as_str) == Some("ctl") { + let rest = std::iter::once(format!("{} ctl", argv[0])).chain(argv.into_iter().skip(2)); + let ctl = CtlArgs::parse_from(rest); + return run_ctl(ctl.socket, ctl.request).await; + } let args = Args::parse(); walshadow::trace::set_sample_ratio(args.trace_sample_ratio); // `--otlp-endpoint` wins; otherwise honor the conventional env var. @@ -403,6 +433,43 @@ async fn main() -> Result<()> { result } +async fn run_ctl(socket: PathBuf, request: Vec) -> Result<()> { + use std::io::{IsTerminal, Read}; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let verb = request.first().map(String::as_str).unwrap_or_default(); + let config: toml::Table = if std::io::stdin().is_terminal() { + toml::Table::new() + } else { + let mut body = String::new(); + std::io::stdin().read_to_string(&mut body)?; + body.parse().context("parse config body as TOML")? + }; + let doc = walshadow::control::encode_request(verb, config)?; + let mut stream = tokio::net::UnixStream::connect(&socket) + .await + .with_context(|| format!("connect control socket {}", socket.display()))?; + stream.write_all(doc.as_bytes()).await?; + stream.flush().await?; + stream.shutdown().await.ok(); + let mut resp = String::new(); + stream.read_to_string(&mut resp).await?; + let first = resp.lines().next().unwrap_or(""); + if let Some(rest) = first.strip_prefix("OK") { + let rest = rest.trim(); + if !rest.is_empty() { + println!("{rest}"); + } + for l in resp.lines().skip(1) { + println!("{l}"); + } + Ok(()) + } else { + eprint!("{resp}"); + std::process::exit(1); + } +} + /// OTLP/gRPC batch tracer provider for `endpoint`. Must run inside the tokio /// runtime (tonic exporter + batch worker need it). fn build_otlp_provider( @@ -484,17 +551,60 @@ fn init_tracing( provider } +fn tget(root: &toml::Table, section: &str, key: &str) -> Option { + match root.get(section)?.as_table()?.get(key)? { + toml::Value::String(s) => Some(s.clone()), + v => Some(v.to_string()), + } +} + +/// `[source]` defaults from the CLI args — the base layer under the config file +/// for connection resolution, shared by the session and the control surface. +fn cli_source_base(args: &Args) -> toml::Table { + let mut s = toml::Table::new(); + s.insert("host".into(), args.host.clone().into()); + s.insert("port".into(), (args.port as i64).into()); + s.insert("user".into(), args.user.clone().into()); + s.insert("dbname".into(), args.dbname.clone().into()); + if let Some(p) = &args.password { + s.insert("password".into(), p.clone().into()); + } + s.insert("sslmode".into(), args.sslmode.clone().into()); + let mut root = toml::Table::new(); + root.insert("source".into(), toml::Value::Table(s)); + root +} + +fn spawn_sighup_reload( + mut sig: tokio::signal::unix::Signal, + reloader: Arc, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + while sig.recv().await.is_some() { + tracing::info!(target: "walshadow", "SIGHUP — live reload"); + if let Err(e) = reloader.reload().await { + tracing::warn!(target: "walshadow", error = %format!("{e:#}"), "reload failed"); + } + } + }) +} + +/// Process-lifetime entry: bind metrics + control socket + SIGHUP, then stream +/// one session. Every reconfigure (socket / SIGHUP) is a live reload — no +/// restart. Ctrl-C breaks the pump loop and drains gracefully. async fn run(args: Args) -> Result<()> { + use walshadow::control::{Reloader, SharedCtx}; + let sighup = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup()) .inspect_err(|e| { tracing::warn!( target: "walshadow::sighup", error = %e, - "SIGHUP install failed; reload disabled", + "SIGHUP install failed", ); })?; // Match systemd SIGTERM with ctrl_c shutdown path - let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + let sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) .inspect_err(|e| { tracing::warn!( target: "walshadow", @@ -502,13 +612,70 @@ async fn run(args: Args) -> Result<()> { "SIGTERM install failed", ); })?; - let sslmode = SslMode::parse(&args.sslmode).context("--sslmode")?; + + let metrics = MetricsRegistry::new(); + let reloader = Arc::new(Reloader::default()); + + let _metrics_server = if let Some(addr) = args.metrics_bind { + let (bound, h) = walshadow::metrics::serve(addr, metrics.clone()) + .await + .context("bind metrics endpoint")?; + tracing::info!(target: "walshadow::metrics", addr = %bound, "metrics endpoint serving"); + Some(h) + } else { + None + }; + + let _control_server = if let Some(sock) = args.control_socket.clone() { + let ch_config = args + .ch_config + .clone() + .context("--control-socket requires --ch-config")?; + let ctx = SharedCtx { + ch_config, + source_base: cli_source_base(&args), + metrics: metrics.clone(), + reloader: reloader.clone(), + frag_lock: Arc::new(Mutex::new(())), + }; + Some( + walshadow::control::serve(sock, ctx) + .await + .context("bind control socket")?, + ) + } else { + None + }; + let _sighup = spawn_sighup_reload(sighup, reloader.clone()); + + run_session(&args, &metrics, &reloader, sigterm).await +} + +async fn run_session( + args: &Args, + metrics: &MetricsRegistry, + reloader: &Arc, + mut sigterm: tokio::signal::unix::Signal, +) -> Result<()> { + // Clone the Arc-backed registry so the body's `&metrics` uses are unchanged. + let metrics = metrics.clone(); + + let merged: toml::Table = match args.ch_config.as_deref() { + Some(p) => walshadow::ch_emitter::load_effective(p, cli_source_base(args)) + .await + .with_context(|| format!("load config {}", p.display()))?, + None => cli_source_base(args), + }; + let sslmode = SslMode::parse(&tget(&merged, "source", "sslmode").unwrap_or_default()) + .context("--sslmode")?; let cfg = PgConfig { - host: args.host.clone(), - port: args.port, - user: args.user.clone(), - password: args.password.clone(), - database: args.dbname.clone(), + host: tget(&merged, "source", "host").unwrap_or_default(), + port: tget(&merged, "source", "port") + .and_then(|v| v.parse().ok()) + .unwrap_or(args.port), + user: tget(&merged, "source", "user").unwrap_or_default(), + password: tget(&merged, "source", "password"), + database: tget(&merged, "source", "dbname").unwrap_or_default(), application_name: "walshadow".into(), sslmode, // Cert/key material rides PGSSL* env, matching libpq (--sslmode doc) @@ -528,11 +695,9 @@ async fn run(args: Args) -> Result<()> { "source identified", ); - let ch_config = if let Some(path) = args.ch_config.as_deref() { - let toml = tokio::fs::read_to_string(path) - .await - .with_context(|| format!("read --ch-config {}", path.display()))?; - let mut cfg = EmitterConfig::from_toml_str(&toml).context("parse --ch-config")?; + // `[ch]` presence decides emitter vs metrics-only. + let ch_config = if merged.contains_key("ch") { + let mut cfg = EmitterConfig::from_table(&merged).context("parse ch config")?; if let Some(ms) = args.ch_flush_timeout_ms { cfg.flush_timeout = std::time::Duration::from_millis(ms); } @@ -547,10 +712,10 @@ async fn run(args: Args) -> Result<()> { // Effective physical replication slot (`[source] slot` + --slot override); // None = slotless. let source_slot: Option = ch_config.as_ref().and_then(|c| c.source_slot.clone()); - let shadow_start = resolve_shadow_start(&args)?; + let shadow_start = resolve_shadow_start(args)?; let bootstrap_end_lsn: Option = if matches!(shadow_start, ShadowStart::Bootstrap(_)) { Some( - run_bootstrap(&cfg, &mut feed, &args, ch_config.clone()) + run_bootstrap(&cfg, &mut feed, args, ch_config.clone()) .await .context("bootstrap")?, ) @@ -563,7 +728,7 @@ async fn run(args: Args) -> Result<()> { let shadow_lifecycle: Option = match &shadow_start { ShadowStart::External => None, ShadowStart::Bootstrap(dir) | ShadowStart::Resume(dir) => { - let shadow = Arc::new(build_owned_shadow(&args, dir.clone())); + let shadow = Arc::new(build_owned_shadow(args, dir.clone())); let conninfo = walsender_primary_conninfo(args.walsender_bind); shadow .write_standby_signal() @@ -960,9 +1125,11 @@ async fn run(args: Args) -> Result<()> { &emitter_cfg, cli_overrides, args.ch_config.clone(), + cli_source_base(args), mapping.clone(), invalidation_epoch.clone(), ); + reloader.set_resolver(Some(resolver.clone())).await; spawn_mapping_refresher(config_rx.clone(), mapping.clone()); // Runtime-config overlay (§7): before the pump consumes WAL, seed the // resolver from source PG's config_* tables via the sidecar libpq @@ -1017,6 +1184,12 @@ async fn run(args: Args) -> Result<()> { let toml_initial_load = emitter_cfg .table_initial_loads .values() + .chain( + emitter_cfg + .table_opt_ins + .values() + .filter_map(|r| r.initial_load.as_ref()), + ) .any(|mode| InitialLoadMode::parse(mode).is_some_and(|m| m != InitialLoadMode::None)); // One validated resident-payload pool for the pipeline and every // concurrent backup pass @@ -1062,6 +1235,21 @@ async fn run(args: Args) -> Result<()> { .with_context(|| format!("seed opt-in for {rel}"))?; } } + for (rel, row) in &emitter_cfg.table_opt_ins { + if row.replicate.is_some() { + walshadow::opt_in::apply_table_opt_in( + &resolver, + &mut applicator, + &catalog, + backfiller_effects.as_ref(), + rel, + row, + raw_start, + ) + .await + .with_context(|| format!("config opt-in for {rel}"))?; + } + } let sql_scoped_tables: HashSet = seeded_table_rows .iter() .filter(|(_, row)| row.replicate.is_some()) @@ -1209,26 +1397,13 @@ async fn run(args: Args) -> Result<()> { .context("open out-dir")?; let mut chunk_buf = Vec::with_capacity(64 * 1024); - let metrics = MetricsRegistry::new(); - let _metrics_server = if let Some(addr) = args.metrics_bind { - let (bound, _handle) = walshadow::metrics::serve(addr, metrics.clone()) - .await - .context("bind metrics endpoint")?; - tracing::info!(target: "walshadow::metrics", addr = %bound, "metrics endpoint serving"); - Some(_handle) - } else { - None - }; - - // Kept for the status loop's config metrics (opt-in / pending-decl gauges); - // the sighup handler takes ownership of `config_resolver` below. + // Metrics endpoint + control socket + SIGHUP are process-lifetime (bound in + // `run`); the session only writes into the shared registry. let metrics_resolver = config_resolver.clone(); let metrics_backfiller = copy_backfiller.clone(); - - // SIGHUP re-reads TOML and republishes the resolved snapshot; the - // mapping refresher + DDL applicator pick it up. Connection params stay - // boot-only. No resolver (metrics-only) makes SIGHUP a no-op tap. - let _sighup_task = spawn_sighup_handler(sighup, config_resolver); + // config_resolver stays owned here (dropped at session end → mapping + // refresher exits); mapping/budget live-reload arrives via the WAL overlay. + let _ = &config_resolver; // Retention sweeper writes shadow's `pg_last_wal_replay_lsn` here; // status loop reads it for the cursor's `shadow_replay_lsn` slot + the @@ -1307,7 +1482,14 @@ async fn run(args: Args) -> Result<()> { let mut last_emitter_ack_observed: u64 = 0; let mut inflight_stall_since: Option = None; let mut inflight_stall_logged = false; + // Pump reads `paused` live off the resolver watch; when paused it idles + // (stops consuming source WAL) without tearing anything down. + let pump_config_rx = config_resolver.as_ref().map(|r| r.subscribe()); let shutdown_reason = loop { + let paused = pump_config_rx + .as_ref() + .map(|rx| rx.borrow().paused) + .unwrap_or(false); // `durable` (fsynced) lags `dispatched`; advertise it as flush/cursor. let dispatched = stream.dispatched_lsn(); let durable = durable_lsn.load(Ordering::Acquire); @@ -1365,16 +1547,17 @@ async fn run(args: Args) -> Result<()> { let dispatched_before = stream.dispatched_lsn(); let chunk = tokio::select! { biased; - // ctrl_c first so it doesn't lose to a chunk already at the queue head. sig = tokio::signal::ctrl_c() => { sig.context("install ctrl_c handler")?; break "signal"; } _ = sigterm.recv() => break "signal", - // Idle tick so metrics/cursor keep tracking the draining pipeline - // when no new WAL arrives. + // Idle tick so metrics/cursor keep tracking, and so a `paused` flip + // is picked up promptly. _ = tokio::time::sleep(metrics_tick) => None, - res = feed.next_chunk(status, &mut chunk_buf) => match res { + // Paused: stop consuming source WAL (idle); resume re-enables this + // arm and the pump continues from the same LSN. + res = feed.next_chunk(status, &mut chunk_buf), if !paused => match res { Ok(Some(c)) => Some(c), Ok(None) => break "CopyDone", Err(e) => { @@ -1801,38 +1984,6 @@ async fn apply_toml_initial_loads( Ok(()) } -/// SIGHUP listener: re-reads `--ch-config` and republishes the resolved -/// snapshot through the resolver (CLI overrides stay on top). Read/parse -/// errors keep the last snapshot in effect; absent resolver (metrics-only) -/// is a no-op tap. -fn spawn_sighup_handler( - mut sig: tokio::signal::unix::Signal, - resolver: Option>, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - loop { - if sig.recv().await.is_none() { - return; - } - let Some(resolver) = resolver.as_ref() else { - tracing::info!(target: "walshadow::sighup", "SIGHUP ignored (no --ch-config)"); - continue; - }; - match resolver.reload().await { - Ok(()) => tracing::info!( - target: "walshadow::sighup", - "ch-config reload published", - ), - Err(e) => tracing::warn!( - target: "walshadow::sighup", - error = %e, - "ch-config reload failed; existing config preserved", - ), - } - } - }) -} - /// Applies each republished [`ResolvedConfig`] snapshot to the live routing /// map. Full swap of the operator mapping, matching the boot seed; runs /// until the resolver's sender drops (SIGHUP disabled or daemon teardown). diff --git a/src/config.rs b/src/config.rs index e0cb45c..d050559 100644 --- a/src/config.rs +++ b/src/config.rs @@ -9,11 +9,12 @@ //! mutated live by [`ConfigResolver::apply_config_event`] as config-table WAL //! writes drain at their commit LSN. `resolve` is the single merge point. //! -//! Connection params (`[ch] host/port/...`) and TOAST stay boot-only fixed -//! points on [`EmitterConfig`]. Everything the operator tunes lives on -//! `ResolvedConfig` and reloads live: per-relation mapping, per-namespace -//! defaults, drop-table strategy, and the emitter batch/compression/retry -//! knobs (read live by the batcher + inserter off the watch channel). +//! Everything the operator tunes lives on `ResolvedConfig` and reloads live: +//! per-relation mapping, per-namespace defaults, drop-table strategy, the +//! emitter batch/compression/retry knobs, the CH connection, and the +//! columns-less table opt-ins — all read off the watch channel (batcher, +//! inserter, DDL applicator, reorder coordinator). Only TOAST + the source +//! connection stay boot-only. //! //! **Storage: in-memory.** The overlay is a derived cache — re-seeded from PG //! then caught up by WAL replay on restart — so it holds no checkpoint. The @@ -68,6 +69,18 @@ pub struct ResolvedConfig { pub compression: CompressionChoice, /// CH client retry budget (live: inserter reads per attempt loop) pub retry_max_attempts: u32, + /// CH connection (live: inserter + DDL applicator reconnect on change). + pub host: String, + pub port: u16, + pub database: String, + pub user: String, + pub password: String, + pub secure: bool, + /// Columns-less `[table.*]` opt-in intents (live: reorder coordinator + /// applies the add/remove diff at a commit barrier). + pub table_opt_ins: HashMap, + /// `[stream] paused` (live: pump idles when true). + pub paused: bool, } impl Default for ResolvedConfig { @@ -144,6 +157,9 @@ struct MergeInputs { pub struct ConfigResolver { /// `--ch-config`; `None` disables reload (nothing to re-read) toml_path: Option, + /// CLI-arg `[source]` base layer, merged under the file on reload (matches + /// boot's `load_effective`). + cli_source_base: toml::Table, cli: CliOverrides, inner: Mutex, tx: watch::Sender>, @@ -175,6 +191,7 @@ impl ConfigResolver { base: &EmitterConfig, cli: CliOverrides, toml_path: Option, + cli_source_base: toml::Table, mapping: MappingHandle, invalidation_epoch: Arc, ) -> (Arc, watch::Receiver>) { @@ -184,6 +201,7 @@ impl ConfigResolver { let (tx, rx) = watch::channel(Arc::new(initial)); let this = Arc::new(Self { toml_path, + cli_source_base, cli, inner: Mutex::new(MergeInputs { base: base.clone(), @@ -447,6 +465,14 @@ impl ConfigResolver { flush_timeout: base.flush_timeout, compression: base.compression, retry_max_attempts: base.retry.max_attempts, + host: base.host.clone(), + port: base.port, + database: base.database.clone(), + user: base.user.clone(), + password: base.password.clone(), + secure: base.secure, + table_opt_ins: base.table_opt_ins.clone(), + paused: base.paused, }; // Runtime-derived layers (before the overlay target loop so a @@ -609,16 +635,16 @@ impl ConfigResolver { (rc, rejections) } - /// Re-read TOML (SIGHUP), re-merge with overlay + CLI, publish. Connection - /// params in the reloaded file are ignored — boot-only. Parse / read - /// errors surface to the caller and leave the last snapshot in effect - /// (watch retains it; no send on failure). + /// Re-read the config (base `--ch-config` + conf.d, CLI-source base under + /// it), re-merge with overlay + CLI, publish. Carries the CH connection + + /// table opt-ins live; the source connection isn't in scope here. Parse / + /// read errors surface to the caller and leave the last snapshot in effect. pub async fn reload(&self) -> Result<(), EmitterError> { let Some(path) = &self.toml_path else { return Ok(()); }; - let toml = tokio::fs::read_to_string(path).await?; - let base = EmitterConfig::from_toml_str(&toml)?; + let merged = crate::ch_emitter::load_effective(path, self.cli_source_base.clone()).await?; + let base = EmitterConfig::from_table(&merged)?; let mut inner = self.inner.lock().await; inner.base = base; self.republish(&inner).await; @@ -954,6 +980,7 @@ mod tests { &base, CliOverrides::default(), None, + toml::Table::new(), mapping.clone(), epoch.clone(), ); @@ -980,8 +1007,14 @@ mod tests { ) .unwrap(); let (mapping, epoch) = dummy_handles(); - let (resolver, mut rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping.clone(), epoch); + let (resolver, mut rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping.clone(), + epoch, + ); let rel = RelName::new("public", "events"); assert!(rx.borrow().tables.contains_key(&rel)); resolver.exclude_table(&rel).await; @@ -998,8 +1031,14 @@ mod tests { async fn derived_mapping_survives_republish() { let base = base_with("retain"); let (mapping, epoch) = dummy_handles(); - let (resolver, mut rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping.clone(), epoch); + let (resolver, mut rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping.clone(), + epoch, + ); let rel = RelName::new("public", "auto"); resolver .register_derived_mapping( @@ -1032,8 +1071,14 @@ mod tests { async fn forget_reparks_opt_in_row_as_pending_decl() { let base = base_with("drop"); let (mapping, epoch) = dummy_handles(); - let (resolver, _rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping.clone(), epoch); + let (resolver, _rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping.clone(), + epoch, + ); let rel = RelName::new("public", "events"); resolver .apply_config_event(ConfigEvent::TableUpserted { @@ -1068,8 +1113,14 @@ mod tests { ) .unwrap(); let (mapping, epoch) = dummy_handles(); - let (resolver, _rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping.clone(), epoch); + let (resolver, _rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping.clone(), + epoch, + ); let mut desc = rel_desc("public", "events"); desc.attributes.push(RelAttr { attnum: 2, @@ -1199,8 +1250,14 @@ mod tests { use crate::runtime_config::ColumnRow; let base = base_with("retain"); let (mapping, epoch) = dummy_handles(); - let (resolver, mut rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping, epoch); + let (resolver, mut rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping, + epoch, + ); let upsert = |ty: &str| ConfigEvent::ColumnUpserted { rel: RelName::new("public", "t"), attname: "amount".into(), @@ -1242,8 +1299,14 @@ mod tests { async fn pending_decl_parks_and_takes() { let base = base_with("retain"); let (mapping, epoch) = dummy_handles(); - let (resolver, _rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping, epoch); + let (resolver, _rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping, + epoch, + ); let rel = RelName::new("app", "later"); resolver .park_pending_decl(rel.clone(), TableRow::default()) @@ -1258,8 +1321,14 @@ mod tests { async fn seed_and_apply_republish() { let base = base_with("retain"); let (mapping, epoch) = dummy_handles(); - let (resolver, mut rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping, epoch); + let (resolver, mut rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping, + epoch, + ); assert_eq!(rx.borrow().drop_table_strategy, "retain"); let overlay = ConfigOverlay { @@ -1284,8 +1353,14 @@ mod tests { async fn reload_without_path_is_noop() { let base = base_with("retain"); let (mapping, epoch) = dummy_handles(); - let (resolver, rx) = - ConfigResolver::new(&base, CliOverrides::default(), None, mapping, epoch); + let (resolver, rx) = ConfigResolver::new( + &base, + CliOverrides::default(), + None, + toml::Table::new(), + mapping, + epoch, + ); resolver.reload().await.unwrap(); assert_eq!(rx.borrow().drop_table_strategy, "retain"); } diff --git a/src/emit/ch_ddl.rs b/src/emit/ch_ddl.rs index 2757b9b..265983c 100644 --- a/src/emit/ch_ddl.rs +++ b/src/emit/ch_ddl.rs @@ -118,8 +118,8 @@ pub struct DdlApplicator { /// and the future overlay retarget DDL without a restart. config_rx: watch::Receiver>, mapping: MappingHandle, - /// Reconnect params, cloned at boot. SIGHUP reloads DDL knobs not - /// connection params, so a reconnect re-dials the boot endpoint. + /// Reconnect params; `refresh_config` updates the connection fields live + /// from a republished snapshot and re-dials on change. conn_cfg: EmitterConfig, retry: RetryConfig, /// Per-attempt cap (shares `EmitterConfig::insert_timeout`); a @@ -199,21 +199,52 @@ impl DdlApplicator { /// strategy). `target_database` + `soft_delete` are boot-only, so they /// carry over. No-op until the resolver sends a new value; called at /// each apply so DDL runs against the current config. - fn refresh_config(&mut self) { - if self.config_rx.has_changed().unwrap_or(false) { + async fn refresh_config(&mut self) -> Result<(), EmitterError> { + if !self.config_rx.has_changed().unwrap_or(false) { + return Ok(()); + } + let (cfg, conn) = { let snap = self.config_rx.borrow_and_update(); - self.config = DdlConfig::from_resolved( + let cfg = DdlConfig::from_resolved( &snap, self.config.target_database.clone(), self.config.soft_delete, ); + let conn = ( + snap.host.clone(), + snap.port, + snap.database.clone(), + snap.user.clone(), + snap.password.clone(), + snap.secure, + ); + (cfg, conn) + }; + self.config = cfg; + let (host, port, database, user, password, secure) = conn; + if host != self.conn_cfg.host + || port != self.conn_cfg.port + || database != self.conn_cfg.database + || user != self.conn_cfg.user + || password != self.conn_cfg.password + || secure != self.conn_cfg.secure + { + self.conn_cfg.host = host; + self.conn_cfg.port = port; + self.conn_cfg.database = database; + self.conn_cfg.user = user; + self.conn_cfg.password = password; + self.conn_cfg.secure = secure; + self.client = connect_client(&self.conn_cfg).await?; + self.last_used = std::time::Instant::now(); } + Ok(()) } /// Errors propagate; the worker task turns them into /// `DecoderSinkError` so the daemon poisons the stream cleanly. pub async fn apply(&mut self, event: &SchemaEvent) -> Result<(), EmitterError> { - self.refresh_config(); + self.refresh_config().await?; match event { SchemaEvent::Added { desc } => self.apply_added(desc).await, SchemaEvent::Changed { old, new, diff } => self.apply_changed(old, new, diff).await, @@ -275,7 +306,7 @@ impl DdlApplicator { /// has no bridgeable shape (nothing created; caller should not map it). /// Idempotent: `IF NOT EXISTS` no-ops a re-create. pub async fn ensure_ch_table(&mut self, desc: &RelDescriptor) -> Result { - self.refresh_config(); + self.refresh_config().await?; let target_db = self .config .target_database_for(&desc.rel_name.namespace) diff --git a/src/emit/ch_emitter.rs b/src/emit/ch_emitter.rs index ccf90a0..8e22fe9 100644 --- a/src/emit/ch_emitter.rs +++ b/src/emit/ch_emitter.rs @@ -37,6 +37,7 @@ use crate::decode::heap_decoder::{ColumnValue, CommittedTuple, HeapOp}; use crate::mapping::{ ColumnMapping, NamespaceMapping, TableMapping, TableTarget, ToastConfig, ToastMode, }; +use crate::runtime_config::TableRow; use crate::schema::{RelDescriptor, RelName}; /// Microseconds between PG `TimestampTz` epoch (2000-01-01 UTC) and Unix @@ -109,6 +110,10 @@ pub struct EmitterConfig { /// Per-table initial-load mode from TOML `[table.*]` blocks. Applies at /// boot for pinned mappings; SQL opt-ins carry their own mode. pub table_initial_loads: HashMap, + pub table_opt_ins: HashMap, + /// `[stream] paused`: pump idles (stops consuming source WAL) when true. + /// Live via reload. + pub paused: bool, /// Per-namespace defaults keyed on PG schema name; per-table /// entries in `tables` win for the relation they name pub namespaces: HashMap, @@ -203,6 +208,8 @@ impl Default for EmitterConfig { flush_timeout: Duration::from_millis(DEFAULT_FLUSH_TIMEOUT_MS), tables: HashMap::new(), table_initial_loads: HashMap::new(), + table_opt_ins: HashMap::new(), + paused: false, namespaces: HashMap::new(), drop_table_strategy: "retain".into(), retry: RetryConfig::default(), @@ -282,9 +289,14 @@ impl EmitterConfig { /// ] /// ``` pub fn from_toml_str(s: &str) -> Result { - use toml::Value; - let root: Value = toml::de::from_str(s) + let root: toml::Table = toml::from_str(s) .map_err(|e: toml::de::Error| EmitterError::Config(format!("toml: {e}")))?; + Self::from_table(&root) + } + + /// Build from an already-parsed (and possibly conf.d-merged) TOML table. + pub fn from_table(root: &toml::Table) -> Result { + use toml::Value; let mut out = Self::default(); if let Some(ch) = root.get("ch").and_then(Value::as_table) { if let Some(v) = ch.get("host").and_then(Value::as_str) { @@ -369,6 +381,14 @@ impl EmitterConfig { // Empty string == omitted == overlay disabled. out.runtime_config_schema = Some(schema.into()); } + if let Some(v) = root + .get("stream") + .and_then(Value::as_table) + .and_then(|t| t.get("paused")) + .and_then(Value::as_bool) + { + out.paused = v; + } if let Some(src) = root.get("source").and_then(Value::as_table) && let Some(slot) = src.get("slot").and_then(Value::as_str) && !slot.is_empty() @@ -416,8 +436,30 @@ impl EmitterConfig { let t = v.as_table().ok_or_else(|| { EmitterError::Config(format!("table.{ns}.{name}: expected a table")) })?; - let replicate = t.get("replicate").and_then(Value::as_bool).unwrap_or(true); - if !replicate { + let replicate = t.get("replicate").and_then(Value::as_bool); + let rel = RelName::new(ns, name); + let Some(cols_v) = t.get("columns").and_then(Value::as_array) else { + out.table_opt_ins.insert( + rel, + TableRow { + target_database: t + .get("target_database") + .and_then(Value::as_str) + .map(String::from), + target_table: t + .get("target_table") + .and_then(Value::as_str) + .map(String::from), + replicate, + initial_load: t + .get("initial_load") + .and_then(Value::as_str) + .map(String::from), + }, + ); + continue; + }; + if replicate == Some(false) { continue; } let database = t @@ -435,9 +477,6 @@ impl EmitterConfig { .and_then(Value::as_str) .unwrap_or(name) .to_string(); - let cols_v = t.get("columns").and_then(Value::as_array).ok_or_else(|| { - EmitterError::Config(format!("table.{ns}.{name}: missing columns array")) - })?; let mut columns = Vec::with_capacity(cols_v.len()); for (i, c) in cols_v.iter().enumerate() { let ct = c.as_table().ok_or_else(|| { @@ -481,7 +520,6 @@ impl EmitterConfig { target_type, }); } - let rel = RelName::new(ns, name); out.tables.insert( rel.clone(), TableMapping { @@ -1423,6 +1461,71 @@ impl std::fmt::Debug for ColumnBuf { } } +/// Load `--ch-config` and deep-merge every `*.toml` in the sibling conf.d +/// directory (`.d/`, e.g. `ch-config.toml` → `ch-config.d/`), in +/// lexical filename order (later wins) — like Postgres `include_dir`. The base +/// file may be absent (empty table); a malformed fragment is a hard error. +pub async fn load_merged(ch_config: &std::path::Path) -> Result { + let mut root: toml::Table = match tokio::fs::read_to_string(ch_config).await { + Ok(s) => toml::from_str(&s).map_err(|e: toml::de::Error| { + EmitterError::Config(format!("parse {}: {e}", ch_config.display())) + })?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => toml::Table::new(), + Err(e) => { + return Err(EmitterError::Config(format!( + "read {}: {e}", + ch_config.display() + ))); + } + }; + let dir = ch_config.with_extension("d"); + if let Ok(mut rd) = tokio::fs::read_dir(&dir).await { + let mut frags: Vec = Vec::new(); + while let Ok(Some(ent)) = rd.next_entry().await { + let p = ent.path(); + if p.extension().and_then(|e| e.to_str()) == Some("toml") { + frags.push(p); + } + } + frags.sort(); + for p in frags { + let s = tokio::fs::read_to_string(&p) + .await + .map_err(|e| EmitterError::Config(format!("read {}: {e}", p.display())))?; + let frag: toml::Table = toml::from_str(&s).map_err(|e: toml::de::Error| { + EmitterError::Config(format!("parse {}: {e}", p.display())) + })?; + merge_tables(&mut root, frag); + } + } + Ok(root) +} + +/// Recursive deep-merge: table-vs-table recurses; any other value from `over` +/// overwrites `base`. +pub fn merge_tables(base: &mut toml::Table, over: toml::Table) { + for (k, v) in over { + match (base.get_mut(&k), v) { + (Some(toml::Value::Table(bt)), toml::Value::Table(ot)) => merge_tables(bt, ot), + (_, v) => { + base.insert(k, v); + } + } + } +} + +/// Effective config: `base` (e.g. the daemon's CLI-arg source defaults) with +/// the on-disk `--ch-config` + conf.d merged over it. Single resolution point +/// shared by the daemon session and the control surface. +pub async fn load_effective( + ch_config: &std::path::Path, + base: toml::Table, +) -> Result { + let mut root = base; + merge_tables(&mut root, load_merged(ch_config).await?); + Ok(root) +} + #[cfg(test)] mod tests { use super::*; @@ -2179,4 +2282,60 @@ mod tests { // Interpolation quotes the dot inside the identifier assert_eq!(dotted_rel.target.sql(), "`default`.`b.c`"); } + + #[test] + fn merge_tables_deep_and_overwrite() { + let mut base: toml::Table = toml::from_str( + "[ch]\nhost = \"base\"\nport = 9000\n[table.\"public.users\"]\ntarget = \"demo.users\"\n", + ) + .unwrap(); + let over: toml::Table = toml::from_str("[ch]\nhost = \"frag\"\n").unwrap(); + merge_tables(&mut base, over); + // fragment overrides [ch].host, keeps [ch].port and the base [table.*]. + assert_eq!( + base["ch"].as_table().unwrap()["host"].as_str(), + Some("frag") + ); + assert_eq!( + base["ch"].as_table().unwrap()["port"].as_integer(), + Some(9000) + ); + assert!(base.get("table").is_some(), "base [table.*] survived"); + } + + #[tokio::test] + async fn load_merged_base_plus_confd_lexical() { + let dir = tempfile::tempdir().unwrap(); + let base = dir.path().join("ch-config.toml"); + let confd = dir.path().join("ch-config.d"); + tokio::fs::write(&base, "[ch]\nhost = \"base\"\nport = 9000\n") + .await + .unwrap(); + tokio::fs::create_dir(&confd).await.unwrap(); + tokio::fs::write(confd.join("10-x.toml"), "[ch]\nhost = \"ten\"\n") + .await + .unwrap(); + tokio::fs::write( + confd.join("50-api.toml"), + "[ch]\nhost = \"fifty\"\ndatabase = \"demo\"\n", + ) + .await + .unwrap(); + let merged = load_merged(&base).await.unwrap(); + let ch = merged["ch"].as_table().unwrap(); + // Higher-numbered fragment wins; base port and fragment database persist. + assert_eq!(ch["host"].as_str(), Some("fifty")); + assert_eq!(ch["port"].as_integer(), Some(9000)); + assert_eq!(ch["database"].as_str(), Some("demo")); + let cfg = EmitterConfig::from_table(&merged).unwrap(); + assert_eq!(cfg.host, "fifty"); + assert_eq!(cfg.database, "demo"); + } + + #[tokio::test] + async fn load_merged_absent_base_ok() { + let dir = tempfile::tempdir().unwrap(); + let merged = load_merged(&dir.path().join("nope.toml")).await.unwrap(); + assert!(merged.is_empty()); + } } diff --git a/src/emit/pipeline/inserter.rs b/src/emit/pipeline/inserter.rs index 8162d1a..276a286 100644 --- a/src/emit/pipeline/inserter.rs +++ b/src/emit/pipeline/inserter.rs @@ -118,16 +118,49 @@ impl Inserter { // held across the reconnect's `&mut self`. let live = self.config_rx.as_ref().map(|rx| { let r = rx.borrow(); - (r.retry_max_attempts, r.compression) + ( + r.retry_max_attempts, + r.compression, + ( + r.host.clone(), + r.port, + r.database.clone(), + r.user.clone(), + r.password.clone(), + r.secure, + ), + ) }); - if let Some((retry_max, compression)) = live { + if let Some((retry_max, compression, (host, port, database, user, password, secure))) = + live + { self.config.retry.max_attempts = retry_max; + // A compression or connection change needs a fresh client (codec + // + socket are fixed at connect) — reconnect at the batch + // boundary, never mid-INSERT. + let mut need_reconnect = false; if compression != self.config.compression { self.config.compression = compression; - if let Err(e) = self.reconnect().await { - fatal.set(format!("inserter compression reconnect: {e}")); - break; - } + need_reconnect = true; + } + if host != self.config.host + || port != self.config.port + || database != self.config.database + || user != self.config.user + || password != self.config.password + || secure != self.config.secure + { + self.config.host = host; + self.config.port = port; + self.config.database = database; + self.config.user = user; + self.config.password = password; + self.config.secure = secure; + need_reconnect = true; + } + if need_reconnect && let Err(e) = self.reconnect().await { + fatal.set(format!("inserter live-config reconnect: {e}")); + break; } } if let Err(e) = self.ensure_asts(&batch.meta) { diff --git a/src/emit/pipeline/reorder.rs b/src/emit/pipeline/reorder.rs index 4acd693..82ece5c 100644 --- a/src/emit/pipeline/reorder.rs +++ b/src/emit/pipeline/reorder.rs @@ -19,7 +19,8 @@ use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use tokio::sync::{Mutex, mpsc, oneshot}; +use std::collections::{HashMap, HashSet}; +use tokio::sync::{Mutex, mpsc, oneshot, watch}; use walrus::pg::walparser::RmId; use crate::catalog::shadow_catalog::{CatalogError, ShadowCatalog}; @@ -28,7 +29,7 @@ use crate::decode::heap_decoder::{DecodedHeap, HeapOp}; use crate::emit::ch_ddl::DdlApplicator; use crate::emit::ch_emitter::EmitterStats; use crate::record::{Record, RecordSink, SinkError}; -use crate::schema::{SchemaEvent, SchemaEventRx}; +use crate::schema::{RelName, SchemaEvent, SchemaEventRx}; use tracing::Instrument; use crate::decode::wal_xact::{ @@ -42,12 +43,12 @@ use crate::xact::xact_buffer::{ drain_pending_schema_events, }; -use crate::config::ConfigResolver; +use crate::config::{ConfigResolver, ResolvedConfig}; use crate::emit::pipeline::Fatal; use crate::emit::pipeline::ack::AckHandle; use crate::emit::pipeline::batcher::BatcherMsg; use crate::emit::pipeline::decode::DecodeJob; -use crate::runtime_config::ConfigEvent; +use crate::runtime_config::{ConfigEvent, TableRow}; use crate::toast::ToastResolver; use crate::toast::toast_retire::RetireLedger; @@ -103,6 +104,17 @@ pub struct ReorderSink { /// OTLP tracing is on; reorder parents `commit.drain`/`dispatch` under /// the `txn` and prunes the entry at commit (the buffer prunes at abort). span_registry: Option, + /// Live-reload receiver + the config-driven opt-in set applied so far. On a + /// republish (`ctl reload` / SIGHUP), the coordinator diffs `table_opt_ins` + /// at the next commit barrier — add → `apply_table_opt_in`, drop → + /// `exclude_table` (CH table retained). + reload_rx: Option>>, + applied_opt_ins: HashSet, + /// Opt-ins whose descriptor the shadow catalog can't resolve yet — a table + /// created just before `ctl tables select` races the CREATE's replay into + /// the shadow. Retried each commit until it resolves, then created + + /// backfilled (moves to `applied_opt_ins`). + pending_opt_ins: HashMap, } impl ReorderSink { @@ -129,6 +141,18 @@ impl ReorderSink { retires: RetireLedger, resume_floor: Arc, ) -> Self { + let reload_rx = config_resolver.as_ref().map(|r| r.subscribe()); + let applied_opt_ins = reload_rx + .as_ref() + .map(|rx| { + rx.borrow() + .table_opt_ins + .iter() + .filter(|(_, row)| row.replicate == Some(true)) + .map(|(rel, _)| rel.clone()) + .collect() + }) + .unwrap_or_default(); Self { buffer, catalog, @@ -151,7 +175,102 @@ impl ReorderSink { span_registry, retires, resume_floor, + reload_rx, + applied_opt_ins, + pending_opt_ins: HashMap::new(), + } + } + + /// Apply a live config reload's table opt-in/opt-out diff at a commit + /// barrier (`opt_in_lsn = commit_lsn`). Base config (mappings/budgets/CH + /// connection) already republished onto the watch; here we do the part that + /// needs the applicator/catalog — create/drop the CH scope. + async fn maybe_apply_reload(&mut self, commit_lsn: u64) -> Result<(), SinkError> { + let Some(resolver) = self.config_resolver.clone() else { + return Ok(()); + }; + // On a republish, re-diff `table_opt_ins`: opt-outs drain now, new + // opt-ins queue as pending, dropped intents leave the queue. + let changed = self + .reload_rx + .as_mut() + .is_some_and(|rx| rx.has_changed().unwrap_or(false)); + if changed { + let desired: Vec<(RelName, TableRow)> = { + let rx = self.reload_rx.as_mut().unwrap(); + let snap = rx.borrow_and_update(); + snap.table_opt_ins + .iter() + .map(|(rel, row)| (rel.clone(), row.clone())) + .collect() + }; + let desired_in: HashSet = desired + .iter() + .filter(|(_, row)| row.replicate == Some(true)) + .map(|(rel, _)| rel.clone()) + .collect(); + let stale: Vec = self + .applied_opt_ins + .iter() + .filter(|rel| !desired_in.contains(*rel)) + .cloned() + .collect(); + for rel in stale { + resolver.exclude_table(&rel).await; + if let Some(b) = &self.backfiller { + b.note_opt_out(&rel).await; + } + self.applied_opt_ins.remove(&rel); + } + self.pending_opt_ins + .retain(|rel, _| desired_in.contains(rel)); + for (rel, row) in desired { + if row.replicate == Some(true) && !self.applied_opt_ins.contains(&rel) { + self.pending_opt_ins.insert(rel, row); + } + } } + // Each commit, apply any pending opt-in the shadow catalog can now + // resolve — a table created just before `select` races the CREATE's + // replay, so retry until the descriptor lands, then create + backfill. + if self.pending_opt_ins.is_empty() { + return Ok(()); + } + let Some(applicator) = self.applicator.as_mut() else { + return Ok(()); + }; + let candidates: Vec<(RelName, TableRow)> = self + .pending_opt_ins + .iter() + .map(|(rel, row)| (rel.clone(), row.clone())) + .collect(); + for (rel, row) in candidates { + let known = self + .catalog + .lock() + .await + .descriptor_by_name(&rel) + .await + .map_err(|e| SinkError::Other(format!("opt-in descriptor lookup: {e}")))? + .is_some(); + if !known { + continue; + } + crate::backfill::opt_in::apply_table_opt_in( + &resolver, + applicator, + &self.catalog, + self.backfiller.as_ref(), + &rel, + &row, + commit_lsn, + ) + .await + .map_err(|e| SinkError::Other(format!("reload opt-in: {e}")))?; + self.pending_opt_ins.remove(&rel); + self.applied_opt_ins.insert(rel); + } + Ok(()) } fn alloc_seq(&mut self) -> u64 { @@ -628,6 +747,9 @@ impl ReorderSink { // while the next loads, so a spilled xact never rematerializes whole. let commit_ts = drain.commit_ts; let commit_lsn = drain.commit_lsn; + // Apply any pending live-reload opt-in/opt-out diff before this commit's + // rows so newly-selected tables are in scope + created for it. + self.maybe_apply_reload(commit_lsn).await?; let mut rows_total: u64 = 0; // Set once a slice's seq registered as publishing (final data slice); // otherwise the trailing rows=0 marker publishes. diff --git a/src/lib.rs b/src/lib.rs index afc6074..fa03c16 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -52,7 +52,7 @@ pub use emit::{ch_ddl, ch_emitter, pipeline}; #[doc(hidden)] pub use filter::{catalog_tracker, classify, filter_segment, main_data, pg_class_decoder, rewrite}; #[doc(hidden)] -pub use ops::{metrics, oracle, preflight, retention, trace}; +pub use ops::{control, metrics, oracle, preflight, retention, trace}; #[doc(hidden)] pub use source::{ manifest, queueing_record_sink, segment_sink, shadow_stream, source_feed, wal_stream, diff --git a/src/ops/control.rs b/src/ops/control.rs new file mode 100644 index 0000000..4e82362 --- /dev/null +++ b/src/ops/control.rs @@ -0,0 +1,657 @@ +//! In-process control plane over a Unix socket +//! +//! TOML bodies preserve config types and let one request update several +//! sections atomically. Mutations only touch `ch-config.d/50-api.toml`, keeping +//! operator-owned config read-only. PeerDB shim consumes this protocol + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, Result, bail}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{UnixListener, UnixStream}; +use tokio::sync::Mutex; +use tokio_postgres::{Client, NoTls}; +use toml::{Table, Value}; + +use crate::metrics::MetricsRegistry; + +/// Holds the running session's resolver so the control socket + SIGHUP can +/// trigger a live `reload()`. The daemon streams one session; there is no +/// start/stop/restart lifecycle — pause is a config flag applied by reload. +#[derive(Default)] +pub struct Reloader { + resolver: Mutex>>, +} + +impl Reloader { + pub async fn set_resolver(&self, r: Option>) { + *self.resolver.lock().await = r; + } + + /// Live reconfigure: re-read the merged config + republish. No restart. + pub async fn reload(&self) -> anyhow::Result<()> { + let r = self.resolver.lock().await.clone(); + if let Some(r) = r { + r.reload() + .await + .map_err(|e| anyhow::anyhow!("reload: {e}"))?; + } + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Shared context handed to the socket handlers +// --------------------------------------------------------------------------- + +/// The managed TOML config path + read handles. No config struct — the file is +/// the source of truth. +#[derive(Clone)] +pub struct SharedCtx { + pub ch_config: PathBuf, + /// CLI-arg `[source]` defaults; the config file overrides them, matching the + /// daemon's connection resolution (see `ch_emitter::load_effective`). + pub source_base: Table, + pub metrics: MetricsRegistry, + pub reloader: Arc, + /// Prevents concurrent fragment updates from overwriting each other + pub frag_lock: Arc>, +} + +// --------------------------------------------------------------------------- +// TOML request protocol +// --------------------------------------------------------------------------- + +/// Keeps CLI and PeerDB shim request framing consistent +pub fn encode_request(verb: &str, config: Table) -> Result { + let body = toml::to_string(&config).context("serialize request config")?; + Ok(format!("{verb}\n{body}")) +} + +pub struct Request<'a> { + pub verb: &'a str, + pub config: Table, +} + +impl<'a> Request<'a> { + /// Preserves TOML types and quoted values across control socket + pub fn parse(buf: &'a [u8]) -> Result> { + let text = std::str::from_utf8(buf).context("request not utf-8")?; + let (head, body) = text.split_once('\n').unwrap_or((text, "")); + let verb = head.split_whitespace().next().context("empty request")?; + let config = if body.trim().is_empty() { + Table::new() + } else { + body.parse().context("parse request config toml")? + }; + Ok(Request { verb, config }) + } +} + +pub fn ok() -> String { + "OK\n".into() +} +pub fn ok_with(body: &str) -> String { + if body.is_empty() { + ok() + } else if body.ends_with('\n') { + format!("OK\n{body}") + } else { + format!("OK\n{body}\n") + } +} +pub fn err(msg: impl std::fmt::Display) -> String { + format!("ERR {msg}\n") +} +fn ok_toml(t: &Table) -> String { + ok_with(&toml::to_string(t).unwrap_or_default()) +} + +// --------------------------------------------------------------------------- +// Socket server +// --------------------------------------------------------------------------- + +/// Bind the control socket (unlinking any stale one, 0600) and serve one +/// request per connection until the runtime tears down. +pub async fn serve(path: PathBuf, ctx: SharedCtx) -> Result> { + if let Some(dir) = path.parent() + && !dir.as_os_str().is_empty() + { + std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?; + } + if let Err(e) = std::fs::remove_file(&path) + && e.kind() != std::io::ErrorKind::NotFound + { + return Err(e).with_context(|| format!("unlink stale {}", path.display())); + } + let listener = UnixListener::bind(&path).with_context(|| format!("bind {}", path.display()))?; + set_mode_600(&path)?; + tracing::info!(target: "walshadow::control", socket = %path.display(), "control socket listening"); + Ok(tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => { + let ctx = ctx.clone(); + tokio::spawn(async move { + if let Err(e) = handle_conn(stream, &ctx).await { + tracing::debug!(target: "walshadow::control", error = %e, "connection errored"); + } + }); + } + Err(e) => tracing::warn!(target: "walshadow::control", error = %e, "accept failed"), + } + } + })) +} + +async fn handle_conn(mut stream: UnixStream, ctx: &SharedCtx) -> std::io::Result<()> { + // EOF framing allows newlines in TOML values + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).await?; + let resp = dispatch(&buf, ctx).await; + stream.write_all(resp.as_bytes()).await?; + stream.flush().await?; + let _ = stream.shutdown().await; + Ok(()) +} + +async fn dispatch(buf: &[u8], ctx: &SharedCtx) -> String { + let req = match Request::parse(buf) { + Ok(r) => r, + Err(e) => return err(format!("{e:#}")), + }; + let res: Result = match req.verb { + "apply" => apply(ctx, &req).await, + "unset" => unset(ctx, &req).await, + "reload" => ctx.reloader.reload().await.map(|()| ok()), + "show" => config_show(ctx).await, + "status" => stream_status(ctx).await, + "tables" => tables_list(ctx, &req).await, + "schemas" => schemas_list(ctx).await, + "columns" => columns_list(ctx, &req).await, + other => Err(anyhow::anyhow!("unknown command {other}")), + }; + res.unwrap_or_else(|e| err(format!("{e:#}"))) +} + +// ---- handlers ------------------------------------------------------------- + +/// Keeps invalid fragments from breaking reloads or later starts +async fn apply(ctx: &SharedCtx, req: &Request<'_>) -> Result { + if req.config.is_empty() { + bail!("empty apply (send a TOML fragment as the body)"); + } + let frag = frag_path(&ctx.ch_config); + let _guard = ctx.frag_lock.lock().await; + let prev = tokio::fs::read(&frag).await.ok(); + let mut root = load(&frag).await?; + crate::ch_emitter::merge_tables(&mut root, req.config.clone()); + save(&frag, &root).await?; + commit_or_rollback(ctx, &frag, prev).await +} + +/// Removes named keys without touching operator-owned base config +async fn unset(ctx: &SharedCtx, req: &Request<'_>) -> Result { + let frag = frag_path(&ctx.ch_config); + let _guard = ctx.frag_lock.lock().await; + let prev = tokio::fs::read(&frag).await.ok(); + let mut root = load(&frag).await?; + apply_mask(&mut root, &req.config); + save(&frag, &root).await?; + commit_or_rollback(ctx, &frag, prev).await +} + +fn apply_mask(root: &mut Table, mask: &Table) { + for (k, v) in mask { + if let Value::Table(sub) = v { + if let Some(Value::Table(t)) = root.get_mut(k) { + apply_mask(t, sub); + } + } else { + root.remove(k); + } + } +} + +/// Restores last valid fragment when validation fails +async fn commit_or_rollback(ctx: &SharedCtx, frag: &Path, prev: Option>) -> Result { + if let Err(e) = validate(ctx).await { + if let Some(bytes) = prev { + tokio::fs::write(frag, bytes).await?; + } else { + tokio::fs::remove_file(frag).await?; + } + return Err(e).context("rejected: merged config invalid"); + } + ctx.reloader.reload().await?; + Ok(ok()) +} + +/// Matches startup validation so accepted fragments remain restart-safe +async fn validate(ctx: &SharedCtx) -> Result<()> { + let merged = get_config(ctx).await?; + crate::ch_emitter::EmitterConfig::from_table(&merged) + .map(|_| ()) + .map_err(|e| anyhow::anyhow!("{e}")) +} + +fn frag_path(ch_config: &Path) -> PathBuf { + ch_config.with_extension("d").join("50-api.toml") +} + +async fn get_config(ctx: &SharedCtx) -> Result { + Ok(crate::ch_emitter::load_effective(&ctx.ch_config, ctx.source_base.clone()).await?) +} + +async fn tables_list<'a>(ctx: &SharedCtx, req: &Request<'a>) -> Result { + let root = get_config(ctx).await?; + let client = pg_connect(&root).await?; + let ns = req.config.get("namespace").and_then(Value::as_str); + let base = "SELECT n.nspname, c.relname, c.relreplident \ + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \ + WHERE c.relkind = 'r' AND n.nspname NOT IN ('pg_catalog','information_schema') \ + AND n.nspname NOT LIKE 'pg\\_%'"; + let rows = if let Some(ns) = ns { + client + .query(&format!("{base} AND n.nspname=$1 ORDER BY 1,2"), &[&ns]) + .await + } else { + client.query(&format!("{base} ORDER BY 1,2"), &[]).await + } + .context("list tables")?; + let selected: std::collections::HashSet<(String, String)> = + selected_tables(&root).into_iter().collect(); + let mut arr = Vec::with_capacity(rows.len()); + for r in rows { + let ns: String = r.get(0); + let rel: String = r.get(1); + let ident: i8 = r.get(2); + let mut t = Table::new(); + t.insert( + "selected".into(), + selected.contains(&(ns.clone(), rel.clone())).into(), + ); + t.insert( + "replica_identity".into(), + Value::String((ident as u8 as char).to_string()), + ); + t.insert("namespace".into(), ns.into()); + t.insert("name".into(), rel.into()); + arr.push(Value::Table(t)); + } + let mut out = Table::new(); + out.insert("tables".into(), Value::Array(arr)); + Ok(ok_toml(&out)) +} + +async fn schemas_list(ctx: &SharedCtx) -> Result { + let root = get_config(ctx).await?; + let client = pg_connect(&root).await?; + let rows = client + .query( + "SELECT nspname FROM pg_namespace \ + WHERE nspname NOT IN ('pg_catalog','information_schema') \ + AND nspname NOT LIKE 'pg\\_%' ORDER BY 1", + &[], + ) + .await + .context("list schemas")?; + let names: Vec = rows.iter().map(|r| r.get::<_, String>(0).into()).collect(); + let mut out = Table::new(); + out.insert("schemas".into(), Value::Array(names)); + Ok(ok_toml(&out)) +} + +async fn columns_list<'a>(ctx: &SharedCtx, req: &Request<'a>) -> Result { + let (Some(ns), Some(rel)) = ( + req.config.get("namespace").and_then(Value::as_str), + req.config.get("relname").and_then(Value::as_str), + ) else { + bail!("usage: columns list with [config] `namespace = \"..\"`, `relname = \"..\"`"); + }; + let root = get_config(ctx).await?; + let client = pg_connect(&root).await?; + let rows = client + .query( + "SELECT a.attname, format_type(a.atttypid, a.atttypmod), a.attnotnull \ + FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid \ + JOIN pg_namespace n ON n.oid=c.relnamespace \ + WHERE n.nspname=$1 AND c.relname=$2 AND a.attnum>0 AND NOT a.attisdropped \ + ORDER BY a.attnum", + &[&ns, &rel], + ) + .await + .context("list columns")?; + let mut arr = Vec::with_capacity(rows.len()); + for r in rows { + let mut t = Table::new(); + t.insert("name".into(), r.get::<_, String>(0).into()); + t.insert("type".into(), r.get::<_, String>(1).into()); + t.insert("notnull".into(), r.get::<_, bool>(2).into()); + arr.push(Value::Table(t)); + } + let mut out = Table::new(); + out.insert("columns".into(), Value::Array(arr)); + Ok(ok_toml(&out)) +} + +/// (namespace, relname) for every `[table..]` block in `root` whose +/// `replicate` isn't `false` (present block = in scope). +fn selected_tables(root: &Table) -> Vec<(String, String)> { + let mut out = Vec::new(); + if let Some(Value::Table(tbl)) = root.get("table") { + for (ns, nsv) in tbl { + if let Value::Table(nst) = nsv { + for (rel, relv) in nst { + if let Value::Table(block) = relv + && block.get("replicate").and_then(Value::as_bool) != Some(false) + { + out.push((ns.clone(), rel.clone())); + } + } + } + } + } + out +} + +async fn stream_status(ctx: &SharedCtx) -> Result { + let paused = get_config(ctx) + .await? + .get("stream") + .and_then(Value::as_table) + .and_then(|t| t.get("paused")) + .and_then(Value::as_bool) + .unwrap_or(false); + let snap = ctx.metrics.snapshot().await; + let mut out = Table::new(); + out.insert("paused".into(), paused.into()); + out.insert( + "rows_synced".into(), + (snap.emitter_rows_total as i64).into(), + ); + out.insert( + "backfills_pending".into(), + (snap.config_backfills_pending as i64).into(), + ); + out.insert( + "lag_bytes".into(), + (snap.shadow_apply_lag_bytes as i64).into(), + ); + out.insert("lag_seconds".into(), snap.shadow_apply_lag_seconds.into()); + out.insert("uptime_secs".into(), (snap.uptime_secs as i64).into()); + Ok(ok_toml(&out)) +} + +async fn config_show(ctx: &SharedCtx) -> Result { + let mut root = get_config(ctx).await?; + for s in ["source", "ch"] { + if let Some(Value::Table(sec)) = root.get_mut(s) + && let Some(p) = sec.get_mut("password") + { + *p = Value::String("***".into()); + } + } + Ok(ok_with(&toml::to_string(&root).unwrap_or_default())) +} + +// ---- TOML file + postgres helpers ----------------------------------------- + +async fn load(path: &Path) -> Result
{ + match tokio::fs::read_to_string(path).await { + Ok(s) => s + .parse::
() + .with_context(|| format!("parse {}", path.display())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Table::new()), + Err(e) => Err(e).with_context(|| format!("read {}", path.display())), + } +} + +async fn save(path: &Path, root: &Table) -> Result<()> { + if let Some(dir) = path.parent() + && !dir.as_os_str().is_empty() + { + tokio::fs::create_dir_all(dir).await.ok(); + } + tokio::fs::write(path, toml::to_string(root).context("serialize toml")?) + .await + .with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +fn render(v: &Value) -> String { + match v { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +fn str_at(root: &Table, section: &str, key: &str) -> String { + root.get(section) + .and_then(Value::as_table) + .and_then(|t| t.get(key)) + .map(render) + .unwrap_or_default() +} + +// TODO: use daemon catalog, direct NoTls connection cannot inspect TLS-only sources +async fn pg_connect(root: &Table) -> Result { + let host = str_at(root, "source", "host"); + if host.is_empty() { + bail!("source host not set"); + } + let mut cfg = tokio_postgres::Config::new(); + cfg.host(&host) + .port(str_at(root, "source", "port").parse().unwrap_or(5432)) + .dbname(nonempty(str_at(root, "source", "dbname"), "postgres")) + .user(nonempty(str_at(root, "source", "user"), "postgres")); + let pw = str_at(root, "source", "password"); + if !pw.is_empty() { + cfg.password(&pw); + } + let (client, conn) = cfg + .connect(NoTls) + .await + .context("connect source postgres")?; + tokio::spawn(async move { + let _ = conn.await; + }); + Ok(client) +} + +// ---- misc ----------------------------------------------------------------- + +fn nonempty(v: String, default: &str) -> String { + if v.is_empty() { default.to_string() } else { v } +} +fn set_mode_600(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .with_context(|| format!("chmod 600 {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg(toml: &str) -> Table { + if toml.is_empty() { + Table::new() + } else { + toml.parse().unwrap() + } + } + + #[test] + fn request_parse() { + // TOML must preserve scalar types and quoted delimiters + let doc = encode_request( + "apply", + cfg("[ch]\nhost = \"db\"\nport = 5432\npassword = \"p a$$=w\""), + ) + .unwrap(); + let r = Request::parse(doc.as_bytes()).unwrap(); + assert_eq!(r.verb, "apply"); + let ch = r.config.get("ch").and_then(Value::as_table).unwrap(); + assert_eq!(ch.get("host").and_then(Value::as_str), Some("db")); + assert_eq!(ch.get("port").and_then(Value::as_integer), Some(5432)); + assert_eq!(ch.get("password").and_then(Value::as_str), Some("p a$$=w")); + + assert!(Request::parse(b"").is_err()); + let r = Request::parse(b"status").unwrap(); + assert_eq!(r.verb, "status"); + assert!(r.config.is_empty()); + } + + #[test] + fn apply_mask_removes_and_recurses() { + let mut root = cfg( + "[source]\nhost = \"h\"\npassword = \"p\"\n[table.demo.a]\nreplicate = true\n[table.demo.b]\nreplicate = true\n", + ); + apply_mask(&mut root, &cfg("[source]\npassword = \"\"")); + assert_eq!(str_at(&root, "source", "host"), "h"); + assert!(root["source"].as_table().unwrap().get("password").is_none()); + apply_mask(&mut root, &cfg("[table.demo]\na = \"\"\nmissing = \"\"")); + let demo = root["table"].as_table().unwrap()["demo"] + .as_table() + .unwrap(); + assert!(demo.get("a").is_none() && demo.get("b").is_some()); + apply_mask(&mut root, &cfg("table = \"\"")); + assert!(root.get("table").is_none()); + } + + fn ctx_at(dir: &Path) -> SharedCtx { + SharedCtx { + ch_config: dir.join("ch-config.toml"), + source_base: Table::new(), + metrics: MetricsRegistry::new(), + reloader: Arc::new(Reloader::default()), + frag_lock: Arc::new(Mutex::new(())), + } + } + + async fn call(sock: &Path, verb: &str, config: &str) -> String { + let doc = encode_request(verb, cfg(config)).unwrap(); + let mut s = UnixStream::connect(sock).await.unwrap(); + s.write_all(doc.as_bytes()).await.unwrap(); + s.shutdown().await.unwrap(); + let mut r = String::new(); + s.read_to_string(&mut r).await.unwrap(); + r + } + + #[tokio::test] + async fn apply_show_status_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let sock = dir.path().join("c.sock"); + let _h = serve(sock.clone(), ctx_at(dir.path())).await.unwrap(); + + assert!( + call( + &sock, + "apply", + "[ch]\nhost = \"ch\"\nport = 9000\ndatabase = \"demo\"\n[stream]\npaused = true" + ) + .await + .starts_with("OK") + ); + // Keep operator-owned base config untouched + assert!(!dir.path().join("ch-config.toml").exists()); + assert!(dir.path().join("ch-config.d/50-api.toml").exists()); + + let shown = call(&sock, "show", "").await; + assert!(shown.contains("host = \"ch\""), "{shown}"); + assert!(shown.contains("paused = true"), "{shown}"); + + assert!( + call(&sock, "apply", "[ch]\npassword = \"secret\"") + .await + .starts_with("OK") + ); + let shown = call(&sock, "show", "").await; + assert!(shown.contains("password = \"***\""), "{shown}"); + assert!(!shown.contains("secret"), "{shown}"); + + let status = call(&sock, "status", "").await; + assert!(status.contains("paused = true"), "{status}"); + let parsed: Table = status.strip_prefix("OK\n").unwrap().parse().unwrap(); + assert_eq!(parsed.get("paused").and_then(Value::as_bool), Some(true)); + assert!(call(&sock, "bogus", "").await.starts_with("ERR")); + assert!(call(&sock, "apply", "").await.starts_with("ERR")); + } + + // Regression: applying one table used to opt every other table out + #[tokio::test] + async fn apply_merges_unset_removes() { + let dir = tempfile::tempdir().unwrap(); + let sock = dir.path().join("c.sock"); + let base = dir.path().join("ch-config.toml"); + std::fs::write( + &base, + "[table.demo.users]\ncolumns = [{ attnum = 1, target = \"id\", type = \"Int64\" }]\n", + ) + .unwrap(); + let _h = serve(sock.clone(), ctx_at(dir.path())).await.unwrap(); + let frag = dir.path().join("ch-config.d/50-api.toml"); + + assert!( + call( + &sock, + "apply", + "[table.demo.gizmos]\nreplicate = true\ninitial_load = \"copy\"" + ) + .await + .starts_with("OK") + ); + let f = std::fs::read_to_string(&frag).unwrap(); + assert!(f.contains("gizmos"), "{f}"); + assert!( + !f.contains("users"), + "apply must not touch the pinned users mapping: {f}" + ); + assert!(f.contains("initial_load = \"copy\""), "{f}"); + + assert!( + call(&sock, "apply", "[table.demo.widgets]\nreplicate = true") + .await + .starts_with("OK") + ); + let f = std::fs::read_to_string(&frag).unwrap(); + assert!(f.contains("gizmos") && f.contains("widgets"), "{f}"); + + assert!( + call(&sock, "unset", "[table.demo]\ngizmos = \"\"") + .await + .starts_with("OK") + ); + let f = std::fs::read_to_string(&frag).unwrap(); + assert!(!f.contains("gizmos") && f.contains("widgets"), "{f}"); + assert!(call(&sock, "unset", "table = \"\"").await.starts_with("OK")); + assert!(!std::fs::read_to_string(&frag).unwrap().contains("widgets")); + // Empty unset is a nop, not an error + assert!(call(&sock, "unset", "").await.starts_with("OK")); + } + + // Invalid fragments must not poison later reloads or starts + #[tokio::test] + async fn apply_rejects_and_rolls_back_invalid() { + let dir = tempfile::tempdir().unwrap(); + let sock = dir.path().join("c.sock"); + let _h = serve(sock.clone(), ctx_at(dir.path())).await.unwrap(); + let frag = dir.path().join("ch-config.d/50-api.toml"); + + assert!( + call(&sock, "apply", "[ch]\nhost = \"ch\"\nport = 9000") + .await + .starts_with("OK") + ); + assert!( + call(&sock, "apply", "[ch]\nport = 70000") + .await + .starts_with("ERR") + ); + let f = std::fs::read_to_string(&frag).unwrap(); + assert!(f.contains("port = 9000") && !f.contains("70000"), "{f}"); + } +} diff --git a/src/ops/mod.rs b/src/ops/mod.rs index b18d6dd..29c85d3 100644 --- a/src/ops/mod.rs +++ b/src/ops/mod.rs @@ -1,3 +1,4 @@ +pub mod control; pub mod metrics; pub mod oracle; pub mod preflight; diff --git a/tests/bootstrap_direct_ch.rs b/tests/bootstrap_direct_ch.rs index 18e0333..192b803 100644 --- a/tests/bootstrap_direct_ch.rs +++ b/tests/bootstrap_direct_ch.rs @@ -187,19 +187,34 @@ async fn direct_bootstrap_ch_end_to_end() { let guard = fx::ChildGuard::new(child); let result = (|| -> Result<()> { - // 7. Wait for the daemon's metrics endpoint. Crossing this - // barrier means: bootstrap finished, daemon-owned shadow is - // serving, preflight passed, WAL pump is in its main loop. - // The bootstrap tail's `wait_through(K)` makes every backfill - // row durable on CH before the daemon hands off to the streaming - // pump, so the 64-row fixture is fully on CH at this barrier. + // 7. Wait for the daemon's metrics endpoint (liveness). The daemon + // binds it before the bootstrap drains to CH, so this is not a + // bootstrap-complete signal on its own. fx::wait_for_listen(metrics_addr, Duration::from_secs(30)) .context("daemon metrics endpoint never came up")?; - // 8. Oracle: count + sum(id) + md5(string_agg(name, ',' ORDER - // BY id)) must match across both sides. The test exercises - // the bootstrap surface; no post-bootstrap workload here, so - // we don't need to drive a `pg_switch_wal` + drain cycle. + // 8. Poll until the bootstrap rows are durable on CH — the tail + // drains asynchronously, so racing it with an immediate assert + // flakes on slow CI. + let src_count = source + .psql_one("SELECT count(*) FROM s14.t") + .context("source count")?; + let deadline = std::time::Instant::now() + Duration::from_secs(60); + loop { + let n = ch + .query("SELECT count() FROM default.t FINAL WHERE _is_deleted = 0") + .unwrap_or_default(); + if n == src_count { + break; + } + if std::time::Instant::now() >= deadline { + anyhow::bail!("bootstrap rows never reached CH: source={src_count}, ch={n}"); + } + std::thread::sleep(Duration::from_millis(200)); + } + + // 9. Oracle: count + sum(id) + md5(string_agg(name, ',' ORDER BY id)) + // must match across both sides. fx::assert_ch_matches_source(&ch, &source, "s14.t", "default.t") .context("source vs CH parity")?; diff --git a/tests/bootstrap_object_store_ch.rs b/tests/bootstrap_object_store_ch.rs index ce0e364..82ddf97 100644 --- a/tests/bootstrap_object_store_ch.rs +++ b/tests/bootstrap_object_store_ch.rs @@ -288,17 +288,35 @@ async fn object_store_bootstrap_ch_end_to_end() { let guard = fx::ChildGuard::new(child); let result = (|| -> Result<()> { - // 8. Wait for daemon's metrics endpoint, bootstrap done, - // daemon-owned shadow live, WAL pump alive. Bootstrap-emitter - // INSERTs flush to CH synchronously before the streaming - // pump starts, so the 64-row fixture lands on CH by this - // point. + // 8. Wait for the daemon's metrics endpoint (liveness). The daemon + // binds it before the bootstrap tail drains to CH, so it is not + // a bootstrap-complete signal on its own. fx::wait_for_listen(metrics_addr, Duration::from_secs(30)) .context("daemon metrics endpoint never came up")?; - // 9. Oracle. ChildGuard's Drop SIGKILLs the daemon at end of - // scope; we don't need a `pg_switch_wal` + drain cycle since - // the test surface is bootstrap correctness, not streaming. + // 9. Poll until the bootstrap rows are durable on CH — the tail + // drains asynchronously, so racing it with an immediate assert + // flakes on slow CI. + let src_count = source + .psql_one("SELECT count(*) FROM s14.t") + .context("source count")?; + let deadline = std::time::Instant::now() + Duration::from_secs(60); + loop { + let n = ch + .query("SELECT count() FROM default.t FINAL WHERE _is_deleted = 0") + .unwrap_or_default(); + if n == src_count { + break; + } + if std::time::Instant::now() >= deadline { + anyhow::bail!("bootstrap rows never reached CH: source={src_count}, ch={n}"); + } + std::thread::sleep(Duration::from_millis(200)); + } + + // 10. Oracle. ChildGuard's Drop SIGKILLs the daemon at end of scope; + // no `pg_switch_wal` + drain cycle since the surface is bootstrap + // correctness, not streaming. fx::assert_ch_matches_source(&ch, &source, "s14.t", "default.t") .context("source vs CH parity")?; diff --git a/tests/common/inproc_harness.rs b/tests/common/inproc_harness.rs index 7437387..a5ced75 100644 --- a/tests/common/inproc_harness.rs +++ b/tests/common/inproc_harness.rs @@ -692,6 +692,7 @@ async fn build_pipeline_inner( &emitter_cfg, walshadow::config::CliOverrides::default(), None, + toml::Table::new(), mapping.clone(), inv_epoch.clone(), ); diff --git a/tests/control_plane_e2e.rs b/tests/control_plane_e2e.rs new file mode 100644 index 0000000..b011c89 --- /dev/null +++ b/tests/control_plane_e2e.rs @@ -0,0 +1,582 @@ +//! Exercises live control changes against real PostgreSQL and ClickHouse +//! +//! Covers pause and reload without restart, table opt-in after catalog replay, +//! and regression where applying one table opted pinned tables out +//! +//! Skipped silently when `initdb`, `pg_basebackup`, or `clickhouse` is +//! absent. Linux only because tests use Unix sockets and POSIX data dirs + +#![cfg(target_os = "linux")] + +#[path = "common/bootstrap_ch_fixture.rs"] +mod fx; + +use std::fs; +use std::net::SocketAddr; +use std::os::unix::process::CommandExt; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, bail}; +use walshadow::shadow::{Shadow, ShadowConfig}; + +struct Ports { + source: u16, + shadow: u16, + ch_tcp: u16, + ch_http: u16, + metrics: u16, + walsender: u16, +} + +// 17400-range: below the ephemeral range, clear of bootstrap_direct_ch +// (17300) and runtime_config_e2e (17700). CH's interserver port is +// ch_http + 1, so metrics/walsender dodge that slot. +const P1: Ports = Ports { + source: 17401, + shadow: 17402, + ch_tcp: 17409, + ch_http: 17410, + metrics: 17415, + walsender: 17416, +}; +const P2: Ports = Ports { + source: 17421, + shadow: 17422, + ch_tcp: 17429, + ch_http: 17430, + metrics: 17435, + walsender: 17436, +}; +const P3: Ports = Ports { + source: 17441, + shadow: 17442, + ch_tcp: 17449, + ch_http: 17450, + metrics: 17455, + walsender: 17456, +}; + +/// Running daemon + its source PG + CH, with the paths the tests poke. +struct Harness { + _tmp: tempfile::TempDir, + source: Shadow, + ch: fx::ChServer, + child: Option, + bin: String, + control_socket: PathBuf, + frag_path: PathBuf, + metrics_addr: SocketAddr, + stderr_path: PathBuf, + shadow_data: PathBuf, + shadow_sock: PathBuf, + shadow_filter_dir: PathBuf, + shadow_port: u16, +} + +impl Harness { + /// Bootstrap source + CH + daemon and block until the daemon's + /// metrics port is up (bootstrap done, shadow serving, WAL pump in + /// its main loop) and the seed row has drained to CH. + async fn up(ports: &Ports) -> Result { + let tmp = tempfile::tempdir().unwrap(); + + // Source PG + schema. demo.users is pinned by the base config, + // so it exists before basebackup and its seed row backfills. + let mut scfg = ShadowConfig::new( + tmp.path().join("source-data"), + tmp.path().join("source-filtered"), + ); + scfg.port = ports.source; + scfg.socket_dir = tmp.path().join("source-sock"); + scfg.ctl_timeout = Duration::from_secs(60); + fs::create_dir_all(&scfg.filter_out_dir).unwrap(); + fs::create_dir_all(&scfg.socket_dir).unwrap(); + let source = Shadow::new(scfg); + source.initdb().context("initdb source")?; + source.write_base_conf().context("source base conf")?; + fx::append_source_conf(&source).context("append source conf")?; + source.start().context("start source")?; + + source + .apply_schema_dump( + "CREATE SCHEMA demo;\n\ + CREATE TABLE demo.users (id bigint PRIMARY KEY, name text NOT NULL, email text NOT NULL);\n\ + ALTER TABLE demo.users REPLICA IDENTITY FULL;\n\ + INSERT INTO demo.users VALUES (1, 'alice', 'alice@seed');\n\ + CHECKPOINT;\n\ + SELECT pg_switch_wal();\n", + ) + .context("source schema")?; + + // CH + pinned dest table for demo.users. + let ch_tmp = tempfile::tempdir().unwrap(); + let ch = fx::ChServer::spawn(ch_tmp, ports.ch_tcp, ports.ch_http).context("spawn ch")?; + ch.query("CREATE DATABASE IF NOT EXISTS demo")?; + ch.query( + "CREATE OR REPLACE TABLE demo.users (\ + id Int64, name String, email String,\ + _lsn UInt64, _xid UInt32,\ + _commit_ts DateTime64(6, 'UTC'), _is_deleted Bool\ + ) ENGINE = ReplacingMergeTree(_lsn, _is_deleted) ORDER BY id", + )?; + + // Base config (read-only-shaped: the API only ever writes the + // conf.d fragment beside it). Pins demo.users by columns. + let ch_config_path = tmp.path().join("ch-config.toml"); + fs::write( + &ch_config_path, + format!( + "[ch]\n\ + host = \"127.0.0.1\"\n\ + port = {}\n\ + database = \"demo\"\n\ + compression = \"lz4\"\n\ + \n\ + [table.demo.users]\n\ + columns = [\n \ + {{ attnum = 1, target = \"id\", type = \"Int64\" }},\n \ + {{ attnum = 2, target = \"name\", type = \"String\" }},\n \ + {{ attnum = 3, target = \"email\", type = \"String\" }},\n\ + ]\n", + ports.ch_tcp, + ), + ) + .context("write base ch-config")?; + let frag_dir = ch_config_path.with_extension("d"); + fs::create_dir_all(&frag_dir).context("create conf.d dir")?; + let frag_path = frag_dir.join("50-api.toml"); + + let shadow_data = tmp.path().join("shadow-data"); + let shadow_sock = tmp.path().join("shadow-sock"); + fs::create_dir_all(&shadow_sock).unwrap(); + let shadow_filter_dir = tmp.path().join("filtered"); + fs::create_dir_all(&shadow_filter_dir).unwrap(); + let spill_dir = tmp.path().join("spill"); + fs::create_dir_all(&spill_dir).unwrap(); + let control_socket = tmp.path().join("control.sock"); + + // Long-lived daemon: no --max-segments, so run_session streams + // forever and the tests drive it live. + let bin = env!("CARGO_BIN_EXE_walshadow-stream").to_string(); + let stderr_path = tmp.path().join("daemon.stderr.log"); + let stderr_file = fs::File::create(&stderr_path).context("open daemon stderr")?; + let metrics_addr: SocketAddr = format!("127.0.0.1:{}", ports.metrics).parse().unwrap(); + let child = Command::new(&bin) + .args([ + "--host", + source.config().socket_dir.to_str().unwrap(), + "--port", + &ports.source.to_string(), + "--user", + "postgres", + "--dbname", + "postgres", + "--sslmode", + "disable", + "--out-dir", + shadow_filter_dir.to_str().unwrap(), + "--shadow-socket-dir", + shadow_sock.to_str().unwrap(), + "--shadow-port", + &ports.shadow.to_string(), + "--shadow-user", + "postgres", + "--shadow-dbname", + "postgres", + "--spill-dir", + spill_dir.to_str().unwrap(), + "--status-interval", + "1", + "--metrics-bind", + &metrics_addr.to_string(), + "--walsender-bind", + &format!("127.0.0.1:{}", ports.walsender), + "--retention-bytes", + "0", + "--ch-config", + ch_config_path.to_str().unwrap(), + "--control-socket", + control_socket.to_str().unwrap(), + "--bootstrap-mode", + "direct", + "--bootstrap-shadow-data-dir", + shadow_data.to_str().unwrap(), + "--bootstrap-shadow-replay-timeout", + "120", + ]) + .env("RUST_LOG", "warn,walshadow=info") + .stdout(Stdio::null()) + .stderr(Stdio::from(stderr_file)) + .process_group(0) + .spawn() + .context("spawn walshadow-stream")?; + + let h = Harness { + _tmp: tmp, + source, + ch, + child: Some(child), + bin, + control_socket, + frag_path, + metrics_addr, + stderr_path, + shadow_data, + shadow_sock, + shadow_filter_dir, + shadow_port: ports.shadow, + }; + + fx::wait_for_listen(h.metrics_addr, Duration::from_secs(60)) + .context("daemon metrics endpoint never came up")?; + // Seed row must be on CH before any drill runs. + h.wait_ch( + "SELECT email FROM demo.users FINAL WHERE _is_deleted = 0 AND id = 1", + "alice@seed", + Duration::from_secs(30), + ) + .await + .context("seed row never reached CH")?; + Ok(h) + } + + /// One `ctl` request against the live socket; returns trimmed stdout. + fn ctl(&self, words: &[&str]) -> Result { + self.ctl_body(words, "") + } + + fn ctl_body(&self, words: &[&str], body: &str) -> Result { + use std::io::Write; + let mut child = Command::new(&self.bin) + .arg("ctl") + .arg("--socket") + .arg(&self.control_socket) + .args(words) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("spawn ctl")?; + child + .stdin + .take() + .context("ctl stdin")? + .write_all(body.as_bytes()) + .context("write ctl body")?; + let out = child.wait_with_output().context("ctl output")?; + if !out.status.success() { + bail!( + "ctl {:?} failed: {}", + words, + String::from_utf8_lossy(&out.stderr) + ); + } + Ok(String::from_utf8_lossy(&out.stdout).trim().to_string()) + } + + fn status_field(&self, key: &str) -> Result { + let body = self.ctl(&["status"])?; + let t: toml::Table = body + .parse() + .with_context(|| format!("parse status toml: {body}"))?; + let v = t + .get(key) + .with_context(|| format!("no {key} in status: {body}"))?; + Ok(match v { + toml::Value::String(s) => s.clone(), + other => other.to_string(), + }) + } + + /// SIGHUP the daemon (triggers `spawn_sighup_reload` → config reload). + fn sighup(&self) -> Result<()> { + let pid = self.child.as_ref().context("daemon gone")?.id(); + let ok = Command::new("kill") + .args(["-HUP", &pid.to_string()]) + .status() + .context("kill -HUP")? + .success(); + if !ok { + bail!("kill -HUP {pid} failed"); + } + Ok(()) + } + + fn psql(&self, sql: &str) -> Result { + Ok(self.source.psql_one(sql)?) + } + + fn ch_get(&self, sql: &str) -> Result { + self.ch.query(sql) + } + + fn alive(&mut self) -> bool { + matches!(self.child.as_mut().map(|c| c.try_wait()), Some(Ok(None))) + } + + /// Poll `sql` until it equals `want` or the deadline passes. + async fn wait_ch(&self, sql: &str, want: &str, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + loop { + let last = self.ch_get(sql).unwrap_or_else(|_| "".into()); + if last == want { + return Ok(()); + } + if Instant::now() >= deadline { + bail!("timeout: want {want:?}, last {last:?} for `{sql}`"); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + + /// Assert `sql` stays `want` for the whole window (the negative case: + /// nothing new flows while paused). + async fn assert_ch_stable(&self, sql: &str, want: &str, window: Duration) -> Result<()> { + let end = Instant::now() + window; + while Instant::now() < end { + let got = self.ch_get(sql)?; + if got != want { + bail!("expected CH frozen at {want:?} but saw {got:?} for `{sql}`"); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + Ok(()) + } + + /// Stop the daemon + the autospawn'd shadow so nothing outlives the + /// tempdir, then return the daemon's (now-flushed) stderr for the + /// caller to fold into a panic on failure. + fn teardown(mut self) -> String { + if let Some(mut c) = self.child.take() { + // SIGINT → graceful drain so tracing flushes to the stderr file; + // SIGKILL only if it doesn't exit promptly. + let _ = Command::new("kill") + .args(["-INT", &c.id().to_string()]) + .status(); + let deadline = Instant::now() + Duration::from_secs(15); + loop { + match c.try_wait() { + Ok(Some(_)) => break, + _ if Instant::now() >= deadline => { + let _ = c.kill(); + let _ = c.wait(); + break; + } + _ => std::thread::sleep(Duration::from_millis(100)), + } + } + } + if self.shadow_data.join("postmaster.pid").exists() { + let mut cfg = + ShadowConfig::new(self.shadow_data.clone(), self.shadow_filter_dir.clone()); + cfg.port = self.shadow_port; + cfg.socket_dir = self.shadow_sock.clone(); + cfg.ctl_timeout = Duration::from_secs(60); + let _ = Shadow::new(cfg).stop(); + } + let _ = self.source.stop(); + fs::read_to_string(&self.stderr_path).unwrap_or_default() + } +} + +fn gated() -> bool { + if !fx::pg_available() { + eprintln!("skip: no initdb on PATH"); + return false; + } + if !fx::pg_basebackup_available() { + eprintln!("skip: no pg_basebackup on PATH"); + return false; + } + if !fx::clickhouse_available() { + eprintln!("skip: no clickhouse binary on PATH"); + return false; + } + true +} + +const USER_EMAIL: &str = + "SELECT argMax(email, _lsn) FROM demo.users WHERE _is_deleted = 0 AND id = 1"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn pause_resume_via_ctl_and_sighup_no_restart() { + if !gated() { + return; + } + let mut h = Harness::up(&P1).await.expect("bring up harness"); + + let result = async { + // Baseline: a WAL update flows to CH. + h.psql("UPDATE demo.users SET email = 'baseline@x' WHERE id = 1")?; + h.wait_ch(USER_EMAIL, "baseline@x", Duration::from_secs(15)) + .await?; + + // --- ctl pause/resume ------------------------------------------- + let uptime_before: u64 = h.status_field("uptime_secs")?.parse().unwrap_or(0); + h.ctl_body(&["apply"], "[stream]\npaused = true")?; + assert_eq!(h.status_field("paused")?, "true", "apply paused → paused"); + // API must only write its own fragment + let frag = fs::read_to_string(&h.frag_path).context("read fragment")?; + assert!(frag.contains("paused = true"), "fragment: {frag}"); + + // Wait past idle tick so next write cannot race pause + tokio::time::sleep(Duration::from_millis(600)).await; + // A write made once paused has settled must not reach CH. + h.psql("UPDATE demo.users SET email = 'while-paused@x' WHERE id = 1")?; + h.assert_ch_stable(USER_EMAIL, "baseline@x", Duration::from_secs(5)) + .await?; + + h.ctl_body(&["apply"], "[stream]\npaused = false")?; + assert_eq!(h.status_field("paused")?, "false", "apply resume → running"); + h.wait_ch(USER_EMAIL, "while-paused@x", Duration::from_secs(15)) + .await?; + + // Uptime catches hidden restarts + let uptime_after: u64 = h.status_field("uptime_secs")?.parse().unwrap_or(0); + assert!( + uptime_after >= uptime_before, + "uptime went backwards ({uptime_before} → {uptime_after}) — daemon restarted", + ); + assert!(h.alive(), "daemon exited during pause/resume"); + + // --- SIGHUP-triggered reload ------------------------------------ + // Fragment changes stay inactive until reload + fs::write(&h.frag_path, "[stream]\npaused = true\n").context("write frag")?; + h.psql("UPDATE demo.users SET email = 'pre-sighup@x' WHERE id = 1")?; + h.wait_ch(USER_EMAIL, "pre-sighup@x", Duration::from_secs(15)) + .await + .context("fragment write alone must not pause the pump")?; + + // SIGHUP applies paused=true; the next write is frozen. + h.sighup()?; + tokio::time::sleep(Duration::from_secs(1)).await; + h.psql("UPDATE demo.users SET email = 'post-sighup@x' WHERE id = 1")?; + h.assert_ch_stable(USER_EMAIL, "pre-sighup@x", Duration::from_secs(5)) + .await + .context("SIGHUP reload did not apply the pause")?; + + // Clear + SIGHUP: resume, the frozen write catches up. + fs::write(&h.frag_path, "[stream]\npaused = false\n").context("write frag")?; + h.sighup()?; + h.wait_ch(USER_EMAIL, "post-sighup@x", Duration::from_secs(15)) + .await + .context("SIGHUP resume did not catch up")?; + + Ok::<(), anyhow::Error>(()) + } + .await; + + let stderr = h.teardown(); + if let Err(e) = result { + panic!("{e:#}\n--- daemon stderr ---\n{stderr}"); + } +} + +#[ignore] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn live_table_opt_in_auto_creates_on_reload() { + if !gated() { + return; + } + let mut h = Harness::up(&P2).await.expect("bring up harness"); + + let result = async { + // An existing table with a pre-opt-in row, absent from CH. + h.psql( + "CREATE TABLE demo.gizmos (id bigint PRIMARY KEY, label text);\ + ALTER TABLE demo.gizmos REPLICA IDENTITY FULL;", + )?; + h.psql("INSERT INTO demo.gizmos VALUES (1, 'alpha')")?; + assert_eq!( + h.ch_get("EXISTS TABLE demo.gizmos")?, + "0", + "gizmos must not exist on CH before opt-in", + ); + + // CREATE may not have reached shadow catalog, trigger commits until it does + h.ctl_body( + &["apply"], + "[table.demo.gizmos]\nreplicate = true\ninitial_load = \"copy\"", + )?; + h.ctl(&["reload"])?; + let mut created = false; + let deadline = Instant::now() + Duration::from_secs(45); + while Instant::now() < deadline { + h.psql("UPDATE demo.users SET email = 'tick@x' WHERE id = 1")?; + if h.ch_get("EXISTS TABLE demo.gizmos").unwrap_or_default() == "1" { + created = true; + break; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + if !created { + bail!("opt-in never auto-created the CH table demo.gizmos"); + } + + // Pre-opt-in row proves copy ran + h.wait_ch( + "SELECT argMax(label, _lsn) FROM demo.gizmos WHERE _is_deleted = 0 AND id = 1", + "alpha", + Duration::from_secs(20), + ) + .await + .context("default backfill did not carry the pre-opt-in row")?; + + h.psql("INSERT INTO demo.gizmos VALUES (2, 'beta')")?; + h.wait_ch( + "SELECT argMax(label, _lsn) FROM demo.gizmos WHERE _is_deleted = 0 AND id = 2", + "beta", + Duration::from_secs(15), + ) + .await + .context("post-opt-in insert did not reach CH")?; + + assert!(h.alive(), "daemon exited during opt-in"); + Ok::<(), anyhow::Error>(()) + } + .await; + + let stderr = h.teardown(); + if let Err(e) = result { + panic!("{e:#}\n--- daemon stderr ---\n{stderr}"); + } +} + +/// Regression: applying one table used to opt pinned tables out +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn apply_preserves_previously_pinned_table() { + if !gated() { + return; + } + let mut h = Harness::up(&P3).await.expect("bring up harness"); + + let result = async { + h.psql("UPDATE demo.users SET email = 'before-select@x' WHERE id = 1")?; + h.wait_ch(USER_EMAIL, "before-select@x", Duration::from_secs(15)) + .await?; + + h.psql( + "CREATE TABLE demo.gadgets (id bigint PRIMARY KEY, label text);\ + ALTER TABLE demo.gadgets REPLICA IDENTITY FULL;", + )?; + h.ctl_body(&["apply"], "[table.demo.gadgets]\nreplicate = true")?; + h.ctl(&["reload"])?; + + // Unrelated apply must preserve pinned mapping + h.psql("UPDATE demo.users SET email = 'after-select@x' WHERE id = 1")?; + h.wait_ch(USER_EMAIL, "after-select@x", Duration::from_secs(15)) + .await + .context("selecting an unrelated table opted demo.users out")?; + + assert!(h.alive(), "daemon exited"); + Ok::<(), anyhow::Error>(()) + } + .await; + + let stderr = h.teardown(); + if let Err(e) = result { + panic!("{e:#}\n--- daemon stderr ---\n{stderr}"); + } +} diff --git a/tests/pgbench_acceptance.rs b/tests/pgbench_acceptance.rs index 2f97d0e..1fb7d8e 100644 --- a/tests/pgbench_acceptance.rs +++ b/tests/pgbench_acceptance.rs @@ -420,15 +420,16 @@ async fn run_ddl_intermix(ports: Ports, decoder_pool: usize, inserter_pool: usiz let guard = fx::ChildGuard::new(child); let result = (|| -> Result<()> { - // 8. Wait for daemon's metrics endpoint. Crossing this barrier - // means: bootstrap finished (≈100k pgbench_accounts rows - // drained to CH), shadow up, WAL pump running. + // 8. Wait for the daemon's metrics endpoint (liveness). The daemon + // binds it before the ≈100k-row bootstrap drains to CH, so it is + // not a bootstrap-complete signal on its own. fx::wait_for_listen(metrics_addr, Duration::from_secs(300)) .context("daemon metrics endpoint never came up")?; // 9. Post-bootstrap row-count parity. pgbench -i -s 1 produces // deterministic counts: 100000 accounts, 1 branch, 10 tellers, - // 0 history rows. + // 0 history rows. The bootstrap drains asynchronously, so poll CH + // until it matches rather than racing an immediate assert. for (src_table, ch_table, expected) in [ ("pgbench_accounts", "default.pgbench_accounts", 100_000_u64), ("pgbench_branches", "default.pgbench_branches", 1), @@ -436,17 +437,25 @@ async fn run_ddl_intermix(ports: Ports, decoder_pool: usize, inserter_pool: usiz ("pgbench_history", "default.pgbench_history", 0), ] { let src = psql_source(&source, &format!("SELECT count(*) FROM {src_table}"))?; - let chc = ch.query(&format!( - "SELECT count() FROM {ch_table} FINAL WHERE _is_deleted = 0" - ))?; anyhow::ensure!( src == expected.to_string(), "source {src_table} count {src} != expected {expected}" ); - anyhow::ensure!( - src == chc, - "bootstrap mismatch {src_table}: source={src}, ch={chc}" - ); + let deadline = std::time::Instant::now() + Duration::from_secs(120); + loop { + let chc = ch + .query(&format!( + "SELECT count() FROM {ch_table} FINAL WHERE _is_deleted = 0" + )) + .unwrap_or_default(); + if chc == src { + break; + } + if std::time::Instant::now() >= deadline { + anyhow::bail!("bootstrap mismatch {src_table}: source={src}, ch={chc}"); + } + std::thread::sleep(Duration::from_millis(300)); + } } // 10. Background pgbench workload. -T 6 wallclock keeps the diff --git a/walshadow-peerdb/Cargo.toml b/walshadow-peerdb/Cargo.toml new file mode 100644 index 0000000..9cd15aa --- /dev/null +++ b/walshadow-peerdb/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "walshadow-peerdb" +version = "0.1.0" +edition = "2024" +description = "PeerDB flow HTTP API shim: grpc-gateway JSON surface translated onto walshadow-control's TOML socket protocol" +license = "AGPL-3.0-only" + +# Thin translator: PeerDB clients on one side, walshadow-control's unix +# socket on the other. Owns zero replication logic; pulls in none of +# walshadow's WAL / native-protocol dependencies. +[[bin]] +name = "walshadow-peerdb" +path = "src/main.rs" + +[lib] +name = "walshadow_peerdb" +path = "src/lib.rs" + +[dependencies] +anyhow = "1" +chrono = { version = "0.4", default-features = false } +clap = { version = "4", features = ["derive", "env"] } +http-body-util = "0.1" +hyper = { version = "1", features = ["http1", "server"] } +hyper-util = { version = "0.1", default-features = false, features = ["tokio"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_urlencoded = "0.7" +toml = { version = "1", default-features = false, features = ["parse", "display", "serde"] } +tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "net", "time", "fs", "sync", "io-util", "signal"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "ansi"] } + +[dev-dependencies] +tempfile = "3" diff --git a/walshadow-peerdb/src/auth.rs b/walshadow-peerdb/src/auth.rs new file mode 100644 index 0000000..9171384 --- /dev/null +++ b/walshadow-peerdb/src/auth.rs @@ -0,0 +1,59 @@ +//! `Authorization` header vs a PEERDB_PASSWORD-style shared secret, +//! mirroring PeerDB gateway behavior: unauthenticated when unset, +//! `Bearer ` prefix stripped, constant-time compare + +use hyper::HeaderMap; + +use crate::error::{Code, GrpcError}; + +pub fn require_auth(password: Option<&str>, headers: &HeaderMap) -> Result<(), GrpcError> { + let Some(password) = password else { + return Ok(()); + }; + let headers: Vec<_> = headers.get_all("authorization").iter().collect(); + let header = match headers.as_slice() { + [] => { + return Err(GrpcError::new( + Code::Unauthenticated, + "missing Authorization header", + )); + } + [one] => *one, + _ => { + return Err(GrpcError::new( + Code::Unauthenticated, + "multiple Authorization headers supplied, request rejected", + )); + } + }; + let value = header.as_bytes(); + let token = value.strip_prefix(b"Bearer ").unwrap_or(value); + if ct_eq(token, password.as_bytes()) { + Ok(()) + } else { + Err(GrpcError::new(Code::Unauthenticated, "invalid token")) + } +} + +fn ct_eq(a: &[u8], b: &[u8]) -> bool { + let mut diff = a.len() ^ b.len(); + for i in 0..a.len().max(b.len()) { + let x = a.get(i).copied().unwrap_or(0); + let y = b.get(i).copied().unwrap_or(0); + diff |= usize::from(x ^ y); + } + diff == 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ct_eq_basics() { + assert!(ct_eq(b"secret", b"secret")); + assert!(!ct_eq(b"secret", b"secret2")); + assert!(!ct_eq(b"", b"secret")); + assert!(ct_eq(b"", b"")); + } +} diff --git a/walshadow-peerdb/src/control.rs b/walshadow-peerdb/src/control.rs new file mode 100644 index 0000000..ceb48fc --- /dev/null +++ b/walshadow-peerdb/src/control.rs @@ -0,0 +1,130 @@ +//! Client side of walshadow-control's TOML socket protocol: one +//! `\n` request per connection, EOF-framed; +//! `OK\n[toml body]` or `ERR \n` back. TOML bodies preserve config +//! types and carry values (passwords) that would break a line protocol + +use std::path::PathBuf; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use toml::{Table, Value}; + +use crate::error::GrpcError; + +#[derive(Debug)] +pub enum ControlError { + /// socket connect / io failure — daemon down + Unavailable(String), + /// daemon lacks the verb + UnknownCommand(String), + /// `ERR ` from the daemon + Daemon(String), +} + +impl From for GrpcError { + fn from(e: ControlError) -> Self { + match e { + ControlError::Unavailable(m) => GrpcError::unavailable(m), + ControlError::UnknownCommand(m) => { + GrpcError::unimplemented(format!("control daemon lacks verb: {m}")) + } + ControlError::Daemon(m) => GrpcError::internal(m), + } + } +} + +#[derive(Clone, Debug)] +pub struct ControlClient { + socket: PathBuf, +} + +impl ControlClient { + pub fn new(socket: PathBuf) -> Self { + Self { socket } + } + + /// Send `\n`, return the parsed OK body as a table + /// (empty when the daemon answered a bare `OK`) + pub async fn call(&self, verb: &str, config: &Table) -> Result { + let body = toml::to_string(config) + .map_err(|e| ControlError::Daemon(format!("serialize request config: {e}")))?; + let req = format!("{verb}\n{body}"); + let mut stream = UnixStream::connect(&self.socket).await.map_err(|e| { + ControlError::Unavailable(format!("control socket {}: {e}", self.socket.display())) + })?; + let io = |e: std::io::Error| ControlError::Unavailable(format!("control io: {e}")); + stream.write_all(req.as_bytes()).await.map_err(io)?; + stream.flush().await.map_err(io)?; + // half-close so the daemon's read_to_end sees EOF + stream.shutdown().await.map_err(io)?; + + let mut resp = String::new(); + stream.read_to_string(&mut resp).await.map_err(io)?; + let (first, rest) = resp.split_once('\n').unwrap_or((resp.as_str(), "")); + if first.trim_end() == "OK" { + rest.parse::
() + .map_err(|e| ControlError::Daemon(format!("parse OK body toml: {e}"))) + } else if let Some(msg) = first.strip_prefix("ERR ") { + if msg.starts_with("unknown command") { + Err(ControlError::UnknownCommand(msg.to_string())) + } else { + Err(ControlError::Daemon(msg.to_string())) + } + } else { + Err(ControlError::Daemon(format!( + "malformed control response: {first:?}" + ))) + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TableRow { + pub namespace: String, + pub relname: String, + pub selected: bool, + pub replica_identity_full: bool, +} + +/// Parse a `tables` reply: `[[tables]]` blocks with `namespace`, `name`, +/// `selected`, `replica_identity` (a `relreplident` char, `f` == full) +pub fn parse_tables(body: &Table) -> Vec { + body.get("tables") + .and_then(Value::as_array) + .map(|arr| { + arr.iter() + .filter_map(|v| { + let t = v.as_table()?; + Some(TableRow { + namespace: t.get("namespace").and_then(Value::as_str)?.to_string(), + relname: t.get("name").and_then(Value::as_str)?.to_string(), + selected: t.get("selected").and_then(Value::as_bool).unwrap_or(false), + replica_identity_full: t.get("replica_identity").and_then(Value::as_str) + == Some("f"), + }) + }) + .collect() + }) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_tables_reply() { + let body: Table = "[[tables]]\nnamespace = \"public\"\nname = \"users\"\nselected = true\nreplica_identity = \"f\"\n\ + [[tables]]\nnamespace = \"public\"\nname = \"orders\"\nselected = false\nreplica_identity = \"d\"\n" + .parse() + .unwrap(); + let rows = parse_tables(&body); + assert_eq!(rows.len(), 2); + assert!(rows[0].selected && rows[0].replica_identity_full); + assert_eq!(rows[1].namespace, "public"); + assert_eq!(rows[1].relname, "orders"); + assert!(!rows[1].selected && !rows[1].replica_identity_full); + // absent `tables` key degrades to empty, not an error + assert!(parse_tables(&Table::new()).is_empty()); + } +} diff --git a/walshadow-peerdb/src/error.rs b/walshadow-peerdb/src/error.rs new file mode 100644 index 0000000..9271ec2 --- /dev/null +++ b/walshadow-peerdb/src/error.rs @@ -0,0 +1,105 @@ +use hyper::StatusCode; + +/// gRPC status codes the shim emits. Wire shape follows grpc-gateway: +/// `{"code": , "message": …, "details": []}` with the gateway's +/// HTTP status mapping +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Code { + InvalidArgument = 3, + NotFound = 5, + AlreadyExists = 6, + FailedPrecondition = 9, + Unimplemented = 12, + Internal = 13, + Unavailable = 14, + Unauthenticated = 16, +} + +impl Code { + pub fn http_status(self) -> StatusCode { + match self { + Code::InvalidArgument | Code::FailedPrecondition => StatusCode::BAD_REQUEST, + Code::NotFound => StatusCode::NOT_FOUND, + Code::AlreadyExists => StatusCode::CONFLICT, + Code::Unimplemented => StatusCode::NOT_IMPLEMENTED, + Code::Internal => StatusCode::INTERNAL_SERVER_ERROR, + Code::Unavailable => StatusCode::SERVICE_UNAVAILABLE, + Code::Unauthenticated => StatusCode::UNAUTHORIZED, + } + } +} + +#[derive(Debug)] +pub struct GrpcError { + pub code: Code, + pub message: String, +} + +impl GrpcError { + pub fn new(code: Code, message: impl Into) -> Self { + Self { + code, + message: message.into(), + } + } + + pub fn invalid(message: impl Into) -> Self { + Self::new(Code::InvalidArgument, message) + } + + pub fn not_found(message: impl Into) -> Self { + Self::new(Code::NotFound, message) + } + + pub fn already_exists(message: impl Into) -> Self { + Self::new(Code::AlreadyExists, message) + } + + pub fn failed_precondition(message: impl Into) -> Self { + Self::new(Code::FailedPrecondition, message) + } + + pub fn unimplemented(message: impl Into) -> Self { + Self::new(Code::Unimplemented, message) + } + + pub fn internal(message: impl Into) -> Self { + Self::new(Code::Internal, message) + } + + pub fn unavailable(message: impl Into) -> Self { + Self::new(Code::Unavailable, message) + } +} + +impl std::fmt::Display for GrpcError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}: {}", self.code, self.message) + } +} + +impl std::error::Error for GrpcError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_mapping_matches_gateway() { + assert_eq!(Code::InvalidArgument.http_status(), StatusCode::BAD_REQUEST); + assert_eq!(Code::NotFound.http_status(), StatusCode::NOT_FOUND); + assert_eq!(Code::AlreadyExists.http_status(), StatusCode::CONFLICT); + assert_eq!( + Code::Unimplemented.http_status(), + StatusCode::NOT_IMPLEMENTED + ); + assert_eq!( + Code::Internal.http_status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + Code::Unavailable.http_status(), + StatusCode::SERVICE_UNAVAILABLE + ); + } +} diff --git a/walshadow-peerdb/src/handlers/mirrors.rs b/walshadow-peerdb/src/handlers/mirrors.rs new file mode 100644 index 0000000..d3ca284 --- /dev/null +++ b/walshadow-peerdb/src/handlers/mirrors.rs @@ -0,0 +1,597 @@ +use std::collections::BTreeMap; + +use serde_json::{Value, json}; +use toml::{Table, Value as TomlValue}; + +use crate::control::parse_tables; +use crate::error::GrpcError; +use crate::handlers::{ + check_dest_identifier, flow_status_from_status, parse_body, rows_synced_from_status, + split_identifier, warn_flow_ignored, warn_mapping_ignored, +}; +use crate::model::{ + CDCBatch, CDCMirrorStatus, CreateCDCFlowRequest, CreateCDCFlowResponse, FlowStateChangeRequest, + FlowStatus, ListMirrorsItem, MirrorStatusRequest, MirrorStatusResponse, TableMapping, +}; +use crate::pb::{EnumToken, now_unix, timestamp_rfc3339}; +use crate::response::Json; +use crate::routes::App; +use crate::state::{MirrorRecord, Role, ShimState, TableRef}; +use crate::warn::warn_ignored; + +/// Resolve source/destination peer names against the registry +fn resolve_peers(state: &ShimState, source: &str, dest: &str) -> Result<(), GrpcError> { + for (name, role) in [(source, Role::Source), (dest, Role::Dest)] { + let record = state + .peers + .get(name) + .ok_or_else(|| GrpcError::not_found(format!("peer {name} not found")))?; + if record.role != role { + return Err(GrpcError::invalid(format!( + "peer {name} has type {}, expected a {} peer", + record.db_type, + match role { + Role::Source => "source Postgres", + Role::Dest => "destination ClickHouse", + } + ))); + } + } + Ok(()) +} + +/// Validate one mapping, splitting the source identifier into a +/// (namespace, relname) pair for the opt-in set +fn opt_in_table(m: &TableMapping) -> Result { + let (namespace, relname) = split_identifier(&m.source_table_identifier)?; + check_dest_identifier(m)?; + warn_mapping_ignored(m); + Ok(TableRef { + namespace: namespace.into(), + relname: relname.into(), + }) +} + +fn reject_unsupported_modes(cfg: &crate::model::FlowConnectionConfigs) -> Result<(), GrpcError> { + // faking success here would make callers believe a load ran + if cfg.resync { + return Err(GrpcError::unimplemented("resync unsupported")); + } + if cfg.initial_snapshot_only { + return Err(GrpcError::unimplemented( + "initialSnapshotOnly unsupported: walshadow is CDC-only", + )); + } + Ok(()) +} + +async fn stream_status(app: &App) -> Result { + app.control + .call("status", &Table::new()) + .await + .map_err(GrpcError::from) +} + +/// `[table..] replicate = true` blocks under a `table` root; empty +/// when no tables (the daemon reads a present block as opted-in) +fn tables_fragment(tables: &[TableRef]) -> Table { + let mut by_ns: BTreeMap<&str, Table> = BTreeMap::new(); + for t in tables { + let mut block = Table::new(); + block.insert("replicate".into(), true.into()); + by_ns + .entry(&t.namespace) + .or_default() + .insert(t.relname.clone(), TomlValue::Table(block)); + } + let mut root = Table::new(); + if !by_ns.is_empty() { + let mut table = Table::new(); + for (ns, rels) in by_ns { + table.insert(ns.into(), TomlValue::Table(rels)); + } + root.insert("table".into(), TomlValue::Table(table)); + } + root +} + +fn set_paused(root: &mut Table, paused: bool) { + let mut stream = Table::new(); + stream.insert("paused".into(), paused.into()); + root.insert("stream".into(), TomlValue::Table(stream)); +} + +/// Reconcile the opt-in set: opt in `desired` (idempotent) then remove +/// `previous` entries no longer wanted. Applying before unsetting keeps a +/// still-wanted table selected at every step, so a live stream never drops +async fn reconcile_tables( + app: &App, + desired: &[TableRef], + previous: &[TableRef], +) -> Result<(), GrpcError> { + let add = tables_fragment(desired); + if !add.is_empty() { + app.control + .call("apply", &add) + .await + .map_err(GrpcError::from)?; + } + let removed: Vec<&TableRef> = previous.iter().filter(|p| !desired.contains(p)).collect(); + if !removed.is_empty() { + let mut by_ns: BTreeMap<&str, Table> = BTreeMap::new(); + for t in &removed { + by_ns + .entry(&t.namespace) + .or_default() + .insert(t.relname.clone(), TomlValue::String(String::new())); + } + let mut table = Table::new(); + for (ns, rels) in by_ns { + table.insert(ns.into(), TomlValue::Table(rels)); + } + let mut mask = Table::new(); + mask.insert("table".into(), TomlValue::Table(table)); + app.control + .call("unset", &mask) + .await + .map_err(GrpcError::from)?; + } + Ok(()) +} + +/// Drop the whole shim-owned `[table]` section from the fragment +async fn clear_tables(app: &App) -> Result<(), GrpcError> { + let mut mask = Table::new(); + mask.insert("table".into(), TomlValue::String(String::new())); + app.control + .call("unset", &mask) + .await + .map_err(GrpcError::from) + .map(|_| ()) +} + +async fn apply_paused(app: &App, paused: bool) -> Result<(), GrpcError> { + let mut root = Table::new(); + set_paused(&mut root, paused); + app.control + .call("apply", &root) + .await + .map_err(GrpcError::from) + .map(|_| ()) +} + +pub async fn validate_cdc(app: &App, v: Value) -> Result, GrpcError> { + let req: CreateCDCFlowRequest = parse_body(v)?; + let Some(cfg) = req.connection_configs else { + return Err(GrpcError::invalid("connectionConfigs required")); + }; + reject_unsupported_modes(&cfg)?; + let state = app.store.get().await; + resolve_peers(&state, &cfg.source_name, &cfg.destination_name)?; + let mut wanted = Vec::new(); + for m in &cfg.table_mappings { + wanted.push(opt_in_table(m)?); + } + // `tables` connects to the (already-applied) source, so reachability and + // table existence validate together; the protocol has no destination probe + let body = app + .control + .call("tables", &Table::new()) + .await + .map_err(|e| match e { + crate::control::ControlError::Daemon(m) => { + GrpcError::invalid(format!("source validation: {m}")) + } + other => other.into(), + })?; + let known = parse_tables(&body); + for t in &wanted { + if !known + .iter() + .any(|k| k.namespace == t.namespace && k.relname == t.relname) + { + return Err(GrpcError::invalid(format!( + "table {}.{} not found on source", + t.namespace, t.relname + ))); + } + } + Ok(Json(json!({}))) +} + +pub async fn create_cdc(app: &App, v: Value) -> Result, GrpcError> { + let req: CreateCDCFlowRequest = parse_body(v.clone())?; + let Some(cfg) = req.connection_configs else { + return Err(GrpcError::invalid("connectionConfigs required")); + }; + if cfg.flow_job_name.is_empty() { + return Err(GrpcError::invalid("flowJobName required")); + } + + let state = app.store.get().await; + if let Some(existing) = &state.mirror { + if existing.name == cfg.flow_job_name && req.attach_to_existing { + return Ok(Json(CreateCDCFlowResponse { + workflow_id: existing.workflow_id.clone(), + })); + } + return Err(GrpcError::already_exists(format!( + "mirror {} already exists; mirror cardinality is one per deployment", + existing.name + ))); + } + + reject_unsupported_modes(&cfg)?; + resolve_peers(&state, &cfg.source_name, &cfg.destination_name)?; + if cfg.table_mappings.is_empty() { + return Err(GrpcError::invalid("tableMappings required")); + } + warn_flow_ignored(&cfg); + let mut tables = Vec::new(); + for m in &cfg.table_mappings { + tables.push(opt_in_table(m)?); + } + + // opt in the tables and unpause in one apply, so the stream never runs + // over an empty selection nor sits selected-but-paused between reloads + let mut frag = tables_fragment(&tables); + set_paused(&mut frag, false); + app.control + .call("apply", &frag) + .await + .map_err(GrpcError::from)?; + + let raw_config = v + .get("connectionConfigs") + .or_else(|| v.get("connection_configs")) + .cloned() + .unwrap_or(Value::Null); + let record = MirrorRecord { + name: cfg.flow_job_name.clone(), + workflow_id: cfg.flow_job_name.clone(), + source_name: cfg.source_name.clone(), + destination_name: cfg.destination_name.clone(), + tables, + do_initial_snapshot: cfg.do_initial_snapshot, + created_at_unix: now_unix(), + config: raw_config, + }; + app.store + .update(|s| { + s.terminated.retain(|n| n != &record.name); + s.mirror = Some(record); + }) + .await + .map_err(|e| GrpcError::internal(format!("persist shim state: {e:#}")))?; + Ok(Json(CreateCDCFlowResponse { + workflow_id: cfg.flow_job_name, + })) +} + +pub async fn state_change(app: &App, v: Value) -> Result, GrpcError> { + let req: FlowStateChangeRequest = parse_body(v)?; + let state = app.store.get().await; + let mirror = match &state.mirror { + Some(m) if m.name == req.flow_job_name => m.clone(), + _ if state.terminated.contains(&req.flow_job_name) + && req.requested_flow_state == FlowStatus::Terminated => + { + return Ok(Json(json!({}))); + } + _ => { + return Err(GrpcError::not_found(format!( + "mirror {} not found", + req.flow_job_name + ))); + } + }; + + if req.drop_mirror_stats { + warn_ignored("dropMirrorStats", "shim keeps no batch history to drop"); + } + + if let Some(update) = req + .flow_config_update + .as_ref() + .and_then(|u| u.cdc_flow_config_update.as_ref()) + { + let mut tables = mirror.tables.clone(); + for m in &update.additional_tables { + let t = opt_in_table(m)?; + if !tables.contains(&t) { + tables.push(t); + } + } + for m in &update.removed_tables { + let (namespace, relname) = split_identifier(&m.source_table_identifier)?; + tables.retain(|t| !(t.namespace == namespace && t.relname == relname)); + } + if update.batch_size != 0 || update.idle_timeout != 0 || update.number_of_syncs != 0 { + warn_ignored( + "cdcFlowConfigUpdate.batching", + "batching governed by walshadow emitter budgets", + ); + } + if !update.updated_env.is_empty() { + warn_ignored( + "cdcFlowConfigUpdate.updatedEnv", + "per-flow env not forwarded", + ); + } + if update.snapshot_num_rows_per_partition != 0 + || update.snapshot_num_partitions_override != 0 + || update.snapshot_max_parallel_workers != 0 + || update.snapshot_num_tables_in_parallel != 0 + { + warn_ignored( + "cdcFlowConfigUpdate.snapshotKnobs", + "backfill knobs have no walshadow counterpart", + ); + } + if update.skip_initial_snapshot_for_table_additions { + warn_ignored( + "skipInitialSnapshotForTableAdditions", + "walshadow always backfills newly opted-in tables", + ); + } + reconcile_tables(app, &tables, &mirror.tables).await?; + app.store + .update(|s| { + if let Some(m) = &mut s.mirror { + m.tables = tables; + } + }) + .await + .map_err(|e| GrpcError::internal(format!("persist shim state: {e:#}")))?; + } + + match req.requested_flow_state { + FlowStatus::Paused => { + apply_paused(app, true).await?; + } + FlowStatus::Running => { + apply_paused(app, false).await?; + } + FlowStatus::Terminated => { + if !req.skip_destination_drop { + // control never drops destination tables; terminate behaves + // as skipDestinationDrop = true always + warn_ignored( + "skipDestinationDrop=false", + "destination tables are never dropped on terminate", + ); + } + // pausing then clearing the opt-in set is idempotent: a + // pause-then-terminate re-pauses without error + apply_paused(app, true).await?; + clear_tables(app).await?; + app.store + .update(|s| { + s.mirror = None; + if !s.terminated.contains(&mirror.name) { + s.terminated.push(mirror.name.clone()); + } + }) + .await + .map_err(|e| GrpcError::internal(format!("persist shim state: {e:#}")))?; + } + // STATUS_UNKNOWN carries a pure config update + FlowStatus::Unknown => {} + other => { + return Err(GrpcError::invalid(format!( + "requestedFlowState {} unsupported", + other.as_str() + ))); + } + } + Ok(Json(json!({}))) +} + +/// One coarse synthetic batch from the rows-synced counter, enough for UI +/// rendering; no per-batch history exists shim-side +fn synth_batches(rows: i64, created_at_unix: i64) -> Vec { + if rows == 0 { + return Vec::new(); + } + vec![CDCBatch { + start_lsn: 0, + end_lsn: 0, + num_rows: rows, + start_time: timestamp_rfc3339(created_at_unix), + end_time: timestamp_rfc3339(now_unix()), + batch_id: 1, + }] +} + +pub async fn mirror_status(app: &App, v: Value) -> Result, GrpcError> { + let req: MirrorStatusRequest = parse_body(v)?; + let state = app.store.get().await; + let Some(mirror) = state + .mirror + .as_ref() + .filter(|m| m.name == req.flow_job_name) + else { + if state.terminated.contains(&req.flow_job_name) { + return Ok(Json(MirrorStatusResponse { + flow_job_name: req.flow_job_name, + cdc_status: CDCMirrorStatus { + config: json!({}), + snapshot_status: json!({"clones": []}), + cdc_batches: Vec::new(), + source_type: "POSTGRES", + destination_type: "CLICKHOUSE", + rows_synced: 0, + }, + current_flow_state: FlowStatus::Terminated, + created_at: String::new(), + })); + } + return Err(GrpcError::not_found(format!( + "mirror {} not found", + req.flow_job_name + ))); + }; + let status = stream_status(app).await?; + let rows = rows_synced_from_status(&status); + Ok(Json(MirrorStatusResponse { + flow_job_name: mirror.name.clone(), + cdc_status: CDCMirrorStatus { + config: mirror.config.clone(), + snapshot_status: json!({"clones": []}), + cdc_batches: if req.exclude_batches { + Vec::new() + } else { + synth_batches(rows, mirror.created_at_unix) + }, + source_type: "POSTGRES", + destination_type: "CLICKHOUSE", + rows_synced: rows, + }, + current_flow_state: flow_status_from_status(&status), + created_at: timestamp_rfc3339(mirror.created_at_unix), + })) +} + +pub async fn list_mirrors(app: &App) -> Result, GrpcError> { + let state = app.store.get().await; + let Some(mirror) = &state.mirror else { + return Ok(Json(json!({"mirrors": []}))); + }; + let status = match stream_status(app).await { + Ok(s) => flow_status_from_status(&s), + // list should render even with control down + Err(_) => FlowStatus::Unknown, + }; + let item = ListMirrorsItem { + id: 1, + workflow_id: mirror.workflow_id.clone(), + name: mirror.name.clone(), + source_name: mirror.source_name.clone(), + source_type: "POSTGRES", + destination_name: mirror.destination_name.clone(), + destination_type: "CLICKHOUSE", + created_at: (mirror.created_at_unix as f64) * 1000.0, + is_cdc: true, + status, + }; + Ok(Json(json!({"mirrors": [item]}))) +} + +pub async fn list_mirror_names(app: &App) -> Json { + let state = app.store.get().await; + let names: Vec<&str> = state.mirror.iter().map(|m| m.name.as_str()).collect(); + Json(json!({"names": names})) +} + +/// Rows counter for the stats endpoints; unknown mirror name → zero rather +/// than an error so UI panels render +async fn mirror_rows(app: &App, flow_job_name: &str) -> (i64, i64) { + let state = app.store.get().await; + let Some(mirror) = state.mirror.as_ref().filter(|m| m.name == flow_job_name) else { + return (0, 0); + }; + let rows = match stream_status(app).await { + Ok(s) => rows_synced_from_status(&s), + Err(_) => 0, + }; + (rows, mirror.created_at_unix) +} + +pub async fn cdc_batches_get(app: &App, flow_job_name: String) -> Json { + let (rows, created) = mirror_rows(app, &flow_job_name).await; + let batches = synth_batches(rows, created); + Json(json!({"cdcBatches": batches, "total": batches.len(), "page": 1})) +} + +pub async fn cdc_batches_post(app: &App, v: Value) -> Json { + let req: crate::model::GetCDCBatchesRequest = parse_body(v).unwrap_or_default(); + let (rows, created) = mirror_rows(app, &req.flow_job_name).await; + let batches = synth_batches(rows, created); + Json(json!({"cdcBatches": batches, "total": batches.len(), "page": 1})) +} + +pub async fn cdc_graph(app: &App, v: Value) -> Json { + let req: crate::model::GraphRequest = parse_body(v).unwrap_or_default(); + let (rows, _) = mirror_rows(app, &req.flow_job_name).await; + let data: Vec = app + .stats + .graph(bucket_secs(&req.aggregate_type)) + .into_iter() + .map(|(time, rows)| json!({"time": time, "rows": rows})) + .collect(); + Json(json!({"data": data, "totalRows": rows.to_string()})) +} + +/// PeerDB TimeAggregateType (number or name) → bucket width in seconds. +/// Defaults to one hour (enum 3), matching the UI's default selection. +fn bucket_secs(agg: &Option) -> i64 { + let n = match agg { + Some(EnumToken::Number(n)) => *n, + Some(EnumToken::Name(s)) => match s.as_str() { + "TIME_AGGREGATE_TYPE_FIVE_MIN" => 1, + "TIME_AGGREGATE_TYPE_FIFTEEN_MIN" => 2, + "TIME_AGGREGATE_TYPE_ONE_DAY" => 4, + "TIME_AGGREGATE_TYPE_ONE_MONTH" => 5, + _ => 3, + }, + None => 3, + }; + match n { + 1 => 300, + 2 => 900, + 4 => 86_400, + 5 => 2_592_000, + _ => 3600, + } +} + +pub async fn table_total_counts(app: &App, flow_job_name: String) -> Json { + let state = app.store.get().await; + let (rows, _) = mirror_rows(app, &flow_job_name).await; + let tables = state + .mirror + .as_ref() + .filter(|m| m.name == flow_job_name) + .map(|m| m.tables.clone()) + .unwrap_or_default(); + // No per-table counter exists daemon-side; a single-table mirror gets the + // exact aggregate, multi-table splits it evenly (remainder on the first). + let n = tables.len() as i64; + let tables_data: Vec = tables + .iter() + .enumerate() + .map(|(i, t)| { + let count = if n == 0 { + 0 + } else { + rows / n + i64::from((i as i64) < rows % n) + }; + json!({ + "tableName": format!("{}.{}", t.namespace, t.relname), + "counts": {"totalCount": count.to_string()}, + }) + }) + .collect(); + Json(json!({ + "totalData": {"totalCount": rows.to_string()}, + "tablesData": tables_data, + })) +} + +pub async fn total_rows_synced(app: &App, flow_job_name: String) -> Json { + let (rows, _) = mirror_rows(app, &flow_job_name).await; + Json(json!({ + "totalCountCDC": rows.to_string(), + "totalCountInitialLoad": "0", + "totalCount": rows.to_string(), + })) +} + +pub async fn initial_load_summary() -> Json { + Json(json!({"tableSummaries": []})) +} + +pub async fn mirror_logs(v: Value) -> Json { + let req: crate::model::ListMirrorLogsRequest = parse_body(v).unwrap_or_default(); + Json(json!({"errors": [], "total": 0, "page": req.page})) +} diff --git a/walshadow-peerdb/src/handlers/misc.rs b/walshadow-peerdb/src/handlers/misc.rs new file mode 100644 index 0000000..4c42b64 --- /dev/null +++ b/walshadow-peerdb/src/handlers/misc.rs @@ -0,0 +1,108 @@ +//! Accept-&-ignore surface: success-shaped empty bodies so PeerDB callers +//! proceed, plus version/instance introspection and the qrep reject + +use hyper::Uri; +use serde_json::{Value, json}; + +use crate::error::GrpcError; +use crate::response::Json; +use crate::routes::App; + +pub async fn version(app: &App) -> Json { + Json(json!({"version": app.version})) +} + +pub async fn instance_info(app: &App) -> Json { + // ready == control socket answers + let status = match app.control.call("status", &toml::Table::new()).await { + Ok(_) => "INSTANCE_STATUS_READY", + Err(_) => "INSTANCE_STATUS_UNKNOWN", + }; + Json(json!({"status": status})) +} + +/// no qrep engine; faking success would make callers believe a load ran +pub async fn qrep_create() -> GrpcError { + GrpcError::unimplemented("qrep flows unsupported: walshadow is CDC-only") +} + +pub async fn alert_configs_get() -> Json { + Json(json!({"configs": []})) +} + +pub async fn alert_config_post() -> Json { + Json(json!({"id": 0})) +} + +pub async fn alert_config_delete() -> Json { + Json(json!({})) +} + +pub async fn dynamic_settings_get() -> Json { + Json(json!({"settings": []})) +} + +pub async fn dynamic_setting_post() -> Json { + Json(json!({})) +} + +pub async fn scripts_get() -> Json { + Json(json!({"scripts": []})) +} + +pub async fn script_post() -> Json { + Json(json!({"id": 0})) +} + +pub async fn script_delete() -> Json { + Json(json!({})) +} + +pub async fn flow_tags_post(v: Value) -> Json { + let name = v + .get("flowName") + .or_else(|| v.get("flow_name")) + .and_then(Value::as_str) + .unwrap_or_default(); + Json(json!({"flowName": name})) +} + +pub async fn flow_tags_get(flow_name: String) -> Json { + Json(json!({"flowName": flow_name, "tags": []})) +} + +pub async fn maintenance_post() -> Json { + Json(json!({"workflowId": "", "runId": ""})) +} + +pub async fn maintenance_status() -> Json { + Json(json!({ + "maintenanceRunning": false, + "phase": "MAINTENANCE_PHASE_UNKNOWN", + "pendingActivities": [], + })) +} + +pub async fn skip_snapshot_wait() -> Json { + Json(json!({"signalSent": false, "message": "walshadow has no snapshot wait"})) +} + +pub async fn sequences_reset() -> Json { + Json(json!({"ok": true, "errorMessage": ""})) +} + +pub async fn cancel_table_addition(v: Value) -> Json { + let name = v + .get("flowJobName") + .or_else(|| v.get("flow_job_name")) + .and_then(Value::as_str) + .unwrap_or_default(); + Json(json!({"flowJobName": name, "tablesAfterCancellation": [], "runId": ""})) +} + +pub async fn unimplemented_fallback(uri: &Uri) -> GrpcError { + GrpcError::unimplemented(format!( + "{} not implemented by walshadow-peerdb", + uri.path() + )) +} diff --git a/walshadow-peerdb/src/handlers/mod.rs b/walshadow-peerdb/src/handlers/mod.rs new file mode 100644 index 0000000..846428f --- /dev/null +++ b/walshadow-peerdb/src/handlers/mod.rs @@ -0,0 +1,342 @@ +pub mod mirrors; +pub mod misc; +pub mod peers; + +use serde::de::DeserializeOwned; +use serde_json::Value as JsonValue; +use toml::{Table, Value}; + +use crate::error::GrpcError; +use crate::model::{ClickhouseConfig, FlowStatus, PostgresConfig, TableMapping}; +use crate::warn::warn_ignored; + +pub fn parse_body(v: JsonValue) -> Result { + serde_json::from_value(v).map_err(|e| GrpcError::invalid(format!("malformed request: {e}"))) +} + +/// Wrap a section table under its config key, ready to `apply` +fn section(key: &str, table: Table) -> Table { + let mut root = Table::new(); + root.insert(key.into(), Value::Table(table)); + root +} + +/// `[source]` apply fragment for a submitted Postgres peer config +pub fn source_fragment(cfg: &PostgresConfig) -> Table { + let port = if cfg.port == 0 { 5432 } else { cfg.port }; + let mut src = Table::new(); + src.insert("host".into(), cfg.host.clone().into()); + src.insert("port".into(), i64::from(port).into()); + src.insert("dbname".into(), cfg.database.clone().into()); + src.insert("user".into(), cfg.user.clone().into()); + src.insert("password".into(), cfg.password.clone().into()); + src.insert("sslmode".into(), cfg.sslmode().into()); + section("source", src) +} + +/// `[ch]` apply fragment for a submitted ClickHouse peer config +pub fn dest_fragment(cfg: &ClickhouseConfig) -> Table { + let port = if cfg.port == 0 { 9000 } else { cfg.port }; + let mut ch = Table::new(); + ch.insert("host".into(), cfg.host.clone().into()); + ch.insert("port".into(), i64::from(port).into()); + ch.insert("database".into(), cfg.database.clone().into()); + ch.insert("user".into(), cfg.user.clone().into()); + ch.insert("password".into(), cfg.password.clone().into()); + ch.insert("secure".into(), (!cfg.disable_tls).into()); + section("ch", ch) +} + +pub fn warn_pg_ignored(cfg: &PostgresConfig) { + if cfg + .metadata_schema + .as_deref() + .is_some_and(|s| !s.is_empty()) + { + warn_ignored( + "postgresConfig.metadataSchema", + "walshadow keeps no catalog metadata schema on source", + ); + } + if cfg.ssh_config.is_some() { + warn_ignored("postgresConfig.sshConfig", "no SSH tunnel support"); + } + if cfg.root_ca.as_deref().is_some_and(|s| !s.is_empty()) { + warn_ignored( + "postgresConfig.rootCa", + "custom CA not forwarded to control", + ); + } + if !cfg.tls_host.is_empty() { + warn_ignored("postgresConfig.tlsHost", "tls host override not forwarded"); + } + if cfg.skip_cert_verification { + warn_ignored( + "postgresConfig.skipCertVerification", + "not forwarded; sslmode covers verification level", + ); + } + if cfg.aws_auth.is_some() { + warn_ignored( + "postgresConfig.awsAuth", + "IAM auth unsupported, password auth only", + ); + } + if cfg + .auth_type + .as_ref() + .is_some_and(|v| *v != 0 && *v != "POSTGRES_PASSWORD") + { + warn_ignored("postgresConfig.authType", "password auth only"); + } +} + +pub fn warn_ch_ignored(cfg: &ClickhouseConfig) { + if !cfg.s3_path.is_empty() + || cfg.s3.is_some() + || !cfg.access_key_id.is_empty() + || !cfg.secret_access_key.is_empty() + || !cfg.region.is_empty() + { + warn_ignored( + "clickhouseConfig.s3", + "walshadow inserts over native protocol, no S3 staging", + ); + } + if cfg.endpoint.as_deref().is_some_and(|s| !s.is_empty()) { + warn_ignored( + "clickhouseConfig.endpoint", + "endpoint override not forwarded", + ); + } + if cfg.certificate.is_some() + || cfg.private_key.is_some() + || cfg.tls_certificate_directory.is_some() + { + warn_ignored( + "clickhouseConfig.clientCert", + "client certificates not forwarded", + ); + } + if cfg.root_ca.as_deref().is_some_and(|s| !s.is_empty()) { + warn_ignored("clickhouseConfig.rootCa", "custom CA not forwarded"); + } + if !cfg.tls_host.is_empty() { + warn_ignored( + "clickhouseConfig.tlsHost", + "tls host override not forwarded", + ); + } + if !cfg.cluster.is_empty() || cfg.replicated { + warn_ignored( + "clickhouseConfig.cluster", + "cluster/replicated DDL not driven by walshadow", + ); + } +} + +pub fn warn_flow_ignored(cfg: &crate::model::FlowConnectionConfigs) { + if !cfg.publication_name.is_empty() { + warn_ignored( + "publicationName", + "walshadow consumes physical WAL, publications don't exist in the model", + ); + } + if !cfg.replication_slot_name.is_empty() { + warn_ignored("replicationSlotName", "physical slot managed by walshadow"); + } + if !cfg.soft_delete_col_name.is_empty() { + warn_ignored( + "softDeleteColName", + "destination shape is walshadow's _lsn convergence model", + ); + } + if !cfg.synced_at_col_name.is_empty() { + warn_ignored( + "syncedAtColName", + "destination shape is walshadow's _lsn convergence model", + ); + } + if cfg.snapshot_num_rows_per_partition != 0 + || cfg.snapshot_num_partitions_override != 0 + || cfg.snapshot_max_parallel_workers != 0 + || cfg.snapshot_num_tables_in_parallel != 0 + || !cfg.snapshot_staging_path.is_empty() + { + warn_ignored( + "snapshotKnobs", + "backfill partitioning/parallelism has no walshadow counterpart", + ); + } + if !cfg.cdc_staging_path.is_empty() { + warn_ignored("cdcStagingPath", "no staging path in walshadow"); + } + if !cfg.env.is_empty() { + warn_ignored("env", "per-flow env not forwarded"); + } + if !cfg.script.is_empty() { + warn_ignored("script", "lua scripting unsupported"); + } + if cfg.system.as_ref().is_some_and(|v| { + !matches!(v, crate::pb::EnumToken::Number(0)) + && !matches!(v, crate::pb::EnumToken::Name(n) if n == "Q") + }) { + warn_ignored("system", "type system fixed to walshadow's PG→CH map"); + } + if cfg.max_batch_size != 0 { + warn_ignored( + "maxBatchSize", + "batching governed by walshadow emitter budgets", + ); + } + if cfg.idle_timeout_seconds != 0 { + warn_ignored( + "idleTimeoutSeconds", + "batching governed by walshadow emitter budgets", + ); + } + if !cfg.do_initial_snapshot { + warn_ignored( + "doInitialSnapshot=false", + "walshadow always backfills newly opted-in tables", + ); + } +} + +pub fn warn_mapping_ignored(m: &TableMapping) { + if !m.exclude.is_empty() { + warn_ignored( + "tableMapping.exclude", + "column exclusion pends runtime-config column overrides", + ); + } + if !m.columns.is_empty() { + warn_ignored( + "tableMapping.columns", + "per-column settings pend runtime-config column overrides", + ); + } + if !m.engine_is_default() { + warn_ignored( + "tableMapping.engine", + "destination engine fixed to ReplacingMergeTree", + ); + } + if !m.partition_key.is_empty() + || !m.sharding_key.is_empty() + || !m.policy_name.is_empty() + || !m.partition_by_expr.is_empty() + { + warn_ignored( + "tableMapping.partitioning", + "partition/sharding/policy overrides pend runtime-config table overrides", + ); + } +} + +/// Source identifiers split into (namespace, relname) at ingress; dotted +/// strings exist only at control-line interpolation +pub fn split_identifier(id: &str) -> Result<(&str, &str), GrpcError> { + id.split_once('.') + .filter(|(ns, rel)| !ns.is_empty() && !rel.is_empty()) + .ok_or_else(|| { + GrpcError::invalid(format!( + "sourceTableIdentifier {id:?} must be namespace.relname" + )) + }) +} + +/// destinationTableIdentifier differing from source naming is rejected +/// until per-table target rename exists in runtime config; bare relname +/// and exact echo both count as matching +pub fn check_dest_identifier(m: &TableMapping) -> Result<(), GrpcError> { + let src = &m.source_table_identifier; + let dst = &m.destination_table_identifier; + if dst.is_empty() || dst == src || Some(dst.as_str()) == src.split_once('.').map(|(_, rel)| rel) + { + return Ok(()); + } + Err(GrpcError::unimplemented(format!( + "destinationTableIdentifier {dst:?} differs from source {src:?}; per-table rename unsupported" + ))) +} + +/// `status` reply → FlowStatus. Paused reflects the config `stream.paused` +/// flag; a pending backfill surfaces as SNAPSHOT. A live daemon always +/// answers running or paused; UNKNOWN is reserved for an unreachable one, +/// which the callers derive from a failed call +pub fn flow_status_from_status(status: &Table) -> FlowStatus { + if status + .get("paused") + .and_then(Value::as_bool) + .unwrap_or(false) + { + FlowStatus::Paused + } else if status + .get("backfills_pending") + .and_then(Value::as_integer) + .unwrap_or(0) + > 0 + { + FlowStatus::Snapshot + } else { + FlowStatus::Running + } +} + +pub fn rows_synced_from_status(status: &Table) -> i64 { + status + .get("rows_synced") + .and_then(Value::as_integer) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn identifier_split() { + assert_eq!( + split_identifier("public.users").unwrap(), + ("public", "users") + ); + assert!(split_identifier("users").is_err()); + assert!(split_identifier(".users").is_err()); + assert!(split_identifier("public.").is_err()); + } + + #[test] + fn dest_identifier_rules() { + let m = |src: &str, dst: &str| TableMapping { + source_table_identifier: src.into(), + destination_table_identifier: dst.into(), + ..Default::default() + }; + assert!(check_dest_identifier(&m("public.users", "")).is_ok()); + assert!(check_dest_identifier(&m("public.users", "public.users")).is_ok()); + assert!(check_dest_identifier(&m("public.users", "users")).is_ok()); + assert!(check_dest_identifier(&m("public.users", "renamed")).is_err()); + } + + #[test] + fn status_mapping() { + let status = |toml: &str| toml.parse::
().unwrap(); + assert_eq!( + flow_status_from_status(&status("paused = false")), + FlowStatus::Running + ); + assert_eq!( + flow_status_from_status(&status("paused = false\nbackfills_pending = 2")), + FlowStatus::Snapshot + ); + assert_eq!( + flow_status_from_status(&status("paused = true")), + FlowStatus::Paused + ); + // absent paused key degrades to running, not unknown + assert_eq!(flow_status_from_status(&Table::new()), FlowStatus::Running); + assert_eq!(rows_synced_from_status(&status("rows_synced = 77")), 77); + assert_eq!(rows_synced_from_status(&Table::new()), 0); + } +} diff --git a/walshadow-peerdb/src/handlers/peers.rs b/walshadow-peerdb/src/handlers/peers.rs new file mode 100644 index 0000000..60584db --- /dev/null +++ b/walshadow-peerdb/src/handlers/peers.rs @@ -0,0 +1,386 @@ +use serde_json::{Value, json}; +use toml::{Table, Value as TomlValue}; + +use crate::control::parse_tables; +use crate::error::GrpcError; +use crate::handlers::{ + dest_fragment, parse_body, source_fragment, warn_ch_ignored, warn_pg_ignored, +}; +use crate::model::{ + ColumnsItem, CreatePeerRequest, CreatePeerResponse, DbType, DropPeerRequest, PeerActivityQuery, + PeerListItem, SchemaTablesQuery, SlotInfo, TableColumnsQuery, TableResponse, + ValidatePeerRequest, ValidatePeerResponse, redact, +}; +use crate::pb::now_unix; +use crate::response::Json; +use crate::routes::App; +use crate::state::{PeerRecord, Role}; + +fn peer_failed(message: impl Into) -> Json { + Json(CreatePeerResponse { + status: "FAILED", + message: message.into(), + }) +} + +/// Role + `apply` fragment from a submitted peer; rejects unsupported types +fn peer_role_and_fragment( + peer: &crate::model::Peer, +) -> Result<(Role, &'static str, Table), GrpcError> { + match ( + &peer.db_type, + &peer.postgres_config, + &peer.clickhouse_config, + ) { + (DbType::Postgres, Some(cfg), _) => { + if cfg.host.is_empty() { + return Err(GrpcError::invalid("postgresConfig.host required")); + } + warn_pg_ignored(cfg); + Ok((Role::Source, "POSTGRES", source_fragment(cfg))) + } + (DbType::Clickhouse, _, Some(cfg)) => { + if cfg.host.is_empty() { + return Err(GrpcError::invalid("clickhouseConfig.host required")); + } + warn_ch_ignored(cfg); + Ok((Role::Dest, "CLICKHOUSE", dest_fragment(cfg))) + } + (t, _, _) => Err(GrpcError::unimplemented(format!( + "peer type {} unsupported; walshadow mirrors Postgres → ClickHouse", + t.as_str() + ))), + } +} + +pub async fn create_peer(app: &App, v: Value) -> Result, GrpcError> { + let req: CreatePeerRequest = parse_body(v.clone())?; + let Some(peer) = req.peer else { + return Ok(peer_failed("peer required")); + }; + if peer.name.is_empty() { + return Ok(peer_failed("peer name required")); + } + let (role, db_type, fragment) = peer_role_and_fragment(&peer)?; + + let state = app.store.get().await; + if let Some(existing) = state.peers.get(&peer.name) { + if existing.role != role { + return Ok(peer_failed(format!( + "peer {} already exists with type {}", + peer.name, existing.db_type + ))); + } + if !req.allow_update { + return Err(GrpcError::already_exists(format!( + "peer {} already exists", + peer.name + ))); + } + } else if let Some((held_by, _)) = state.peer_by_role(role) { + // one control daemon drives one streamer: a single source and a + // single destination slot + return Ok(peer_failed(format!( + "{} slot already held by peer {held_by}; one walshadow deployment per pipe", + match role { + Role::Source => "source", + Role::Dest => "destination", + } + ))); + } + + app.control + .call("apply", &fragment) + .await + .map_err(GrpcError::from)?; + let raw_peer = v.get("peer").cloned().unwrap_or(Value::Null); + app.store + .update(|s| { + s.peers.insert( + peer.name.clone(), + PeerRecord { + db_type: db_type.into(), + role, + config: raw_peer, + created_at_unix: now_unix(), + }, + ); + }) + .await + .map_err(|e| GrpcError::internal(format!("persist shim state: {e:#}")))?; + Ok(Json(CreatePeerResponse { + status: "CREATED", + message: String::new(), + })) +} + +/// The TOML protocol has no non-persisting connection probe (`apply` would +/// mutate and reload the daemon), so validation is structural: the request +/// must carry a supported peer type with a host. Connectivity surfaces when +/// the config is applied by `create_peer`, or on `mirrors/cdc/validate`, +/// which lists source tables over the live socket +pub async fn validate_peer(v: Value) -> Result, GrpcError> { + let req: ValidatePeerRequest = parse_body(v)?; + let Some(peer) = req.peer else { + return Err(GrpcError::invalid("peer required")); + }; + let invalid = match ( + &peer.db_type, + &peer.postgres_config, + &peer.clickhouse_config, + ) { + (DbType::Postgres, Some(cfg), _) if cfg.host.is_empty() => { + Some("postgresConfig.host required".into()) + } + (DbType::Postgres, Some(_), _) => None, + (DbType::Clickhouse, _, Some(cfg)) if cfg.host.is_empty() => { + Some("clickhouseConfig.host required".into()) + } + (DbType::Clickhouse, _, Some(_)) => None, + (t, _, _) => Some(format!("peer type {} unsupported", t.as_str())), + }; + Ok(Json(match invalid { + None => ValidatePeerResponse { + status: "VALID", + message: String::new(), + }, + Some(message) => ValidatePeerResponse { + status: "INVALID", + message, + }, + })) +} + +pub async fn drop_peer(app: &App, v: Value) -> Result, GrpcError> { + let req: DropPeerRequest = parse_body(v)?; + let state = app.store.get().await; + if !state.peers.contains_key(&req.peer_name) { + return Err(GrpcError::not_found(format!( + "peer {} not found", + req.peer_name + ))); + } + if let Some(m) = &state.mirror + && (m.source_name == req.peer_name || m.destination_name == req.peer_name) + { + return Err(GrpcError::failed_precondition(format!( + "peer {} is referenced by mirror {}", + req.peer_name, m.name + ))); + } + app.store + .update(|s| { + s.peers.remove(&req.peer_name); + }) + .await + .map_err(|e| GrpcError::internal(format!("persist shim state: {e:#}")))?; + Ok(Json(json!({}))) +} + +pub async fn list_peers(app: &App) -> Json { + let state = app.store.get().await; + let items: Vec<_> = state + .peers + .iter() + .map(|(name, p)| { + serde_json::to_value(PeerListItem { + name, + db_type: &p.db_type, + }) + .unwrap_or_default() + }) + .collect(); + let by_role = |role: Role| -> Vec { + state + .peers + .iter() + .filter(|(_, p)| p.role == role) + .map(|(name, p)| { + serde_json::to_value(PeerListItem { + name, + db_type: &p.db_type, + }) + .unwrap_or_default() + }) + .collect() + }; + Json(json!({ + "items": items, + "sourceItems": by_role(Role::Source), + "destinationItems": by_role(Role::Dest), + })) +} + +fn get_peer(state: &crate::state::ShimState, name: &str) -> Result { + state + .peers + .get(name) + .cloned() + .ok_or_else(|| GrpcError::not_found(format!("peer {name} not found"))) +} + +pub async fn peer_info(app: &App, peer_name: String) -> Result, GrpcError> { + let state = app.store.get().await; + let record = get_peer(&state, &peer_name)?; + let mut peer = record.config; + redact(&mut peer); + Ok(Json(json!({"peer": peer, "version": ""}))) +} + +pub async fn peer_type(app: &App, peer_name: String) -> Result, GrpcError> { + let state = app.store.get().await; + let record = get_peer(&state, &peer_name)?; + Ok(Json(json!({"peerType": record.db_type}))) +} + +/// Source peer must exist and hold the source role before introspection +async fn require_source(app: &App, peer_name: &str) -> Result<(), GrpcError> { + let state = app.store.get().await; + let record = get_peer(&state, peer_name)?; + if record.role != Role::Source { + return Err(GrpcError::invalid(format!( + "peer {peer_name} is not a source Postgres peer" + ))); + } + Ok(()) +} + +pub async fn schemas(app: &App, q: PeerActivityQuery) -> Result, GrpcError> { + require_source(app, &q.peer_name).await?; + let body = app + .control + .call("schemas", &Table::new()) + .await + .map_err(GrpcError::from)?; + let schemas: Vec<&str> = body + .get("schemas") + .and_then(TomlValue::as_array) + .map(|a| a.iter().filter_map(TomlValue::as_str).collect()) + .unwrap_or_default(); + Ok(Json(json!({"schemas": schemas}))) +} + +/// `tables`, scoped to one namespace when asked; the daemon filters +async fn list_tables( + app: &App, + namespace: Option<&str>, +) -> Result, GrpcError> { + let mut req = Table::new(); + if let Some(ns) = namespace { + req.insert("namespace".into(), ns.into()); + } + let body = app + .control + .call("tables", &req) + .await + .map_err(GrpcError::from)?; + Ok(parse_tables(&body)) +} + +pub async fn tables_in_schema(app: &App, q: SchemaTablesQuery) -> Result, GrpcError> { + require_source(app, &q.peer_name).await?; + let rows = list_tables(app, Some(&q.schema_name)).await?; + let tables: Vec<_> = rows + .into_iter() + .map(|t| TableResponse { + table_name: t.relname, + can_mirror: true, + table_size: String::new(), + is_replica_identity_full: t.replica_identity_full, + }) + .collect(); + Ok(Json(json!({"tables": tables}))) +} + +pub async fn all_tables(app: &App, q: PeerActivityQuery) -> Result, GrpcError> { + require_source(app, &q.peer_name).await?; + let rows = list_tables(app, None).await?; + let tables: Vec = rows + .into_iter() + .map(|t| format!("{}.{}", t.namespace, t.relname)) + .collect(); + Ok(Json(json!({"tables": tables}))) +} + +pub async fn columns(app: &App, q: TableColumnsQuery) -> Result, GrpcError> { + require_source(app, &q.peer_name).await?; + let mut req = Table::new(); + req.insert("namespace".into(), q.schema_name.clone().into()); + req.insert("relname".into(), q.table_name.clone().into()); + let body = app + .control + .call("columns", &req) + .await + .map_err(GrpcError::from)?; + // `columns` carries name/type/notnull; the protocol exposes no per-column + // key membership, so is_key / is_replica_identity stay false + let columns: Vec<_> = body + .get("columns") + .and_then(TomlValue::as_array) + .map(|arr| { + arr.iter() + .filter_map(|v| { + let t = v.as_table()?; + Some(ColumnsItem { + name: t.get("name").and_then(TomlValue::as_str)?.to_string(), + column_type: t + .get("type") + .and_then(TomlValue::as_str) + .unwrap_or_default() + .to_string(), + is_key: false, + qkind: String::new(), + is_replica_identity: false, + }) + }) + .collect() + }) + .unwrap_or_default(); + Ok(Json(json!({"columns": columns}))) +} + +pub async fn slots(app: &App, peer_name: String) -> Result, GrpcError> { + require_source(app, &peer_name).await?; + let status = app + .control + .call("status", &Table::new()) + .await + .map_err(GrpcError::from)?; + let lag_bytes = status + .get("lag_bytes") + .and_then(TomlValue::as_integer) + .unwrap_or(0) as f32; + // physical slot presented in logical-slot clothing + let slot = SlotInfo { + slot_name: "walshadow".into(), + redo_lsn: String::new(), + restart_lsn: String::new(), + active: !status + .get("paused") + .and_then(TomlValue::as_bool) + .unwrap_or(true), + lag_in_mb: lag_bytes / (1024.0 * 1024.0), + confirmed_flush_lsn: String::new(), + wal_status: "reserved".into(), + }; + Ok(Json(json!({"slotData": [slot]}))) +} + +pub async fn stats(app: &App, peer_name: String) -> Result, GrpcError> { + require_source(app, &peer_name).await?; + Ok(Json(json!({"statData": []}))) +} + +/// walshadow consumes physical WAL; publications don't exist in the model +pub async fn publications() -> Json { + Json(json!({"publicationNames": []})) +} + +/// Serving empty disables UI type pickers, the safe start +pub async fn all_type_conversions() -> Json { + Json(json!({"conversions": []})) +} + +pub async fn slot_lag_history() -> Json { + Json(json!({"data": []})) +} diff --git a/walshadow-peerdb/src/lib.rs b/walshadow-peerdb/src/lib.rs new file mode 100644 index 0000000..a2ffe21 --- /dev/null +++ b/walshadow-peerdb/src/lib.rs @@ -0,0 +1,11 @@ +pub mod auth; +pub mod control; +pub mod error; +pub mod handlers; +pub mod model; +pub mod pb; +pub mod response; +pub mod routes; +pub mod state; +pub mod stats; +pub mod warn; diff --git a/walshadow-peerdb/src/main.rs b/walshadow-peerdb/src/main.rs new file mode 100644 index 0000000..27bf966 --- /dev/null +++ b/walshadow-peerdb/src/main.rs @@ -0,0 +1,106 @@ +use std::convert::Infallible; +use std::path::PathBuf; +use std::pin::pin; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use clap::Parser; +use hyper::service::service_fn; +use hyper_util::rt::TokioIo; + +use walshadow_peerdb::control::ControlClient; +use walshadow_peerdb::routes::{App, handle}; +use walshadow_peerdb::state::Store; + +/// PeerDB flow HTTP API shim over walshadow-control. Default bind matches +/// PeerDB's HTTP gateway port so existing client config carries over +#[derive(Parser)] +#[command(name = "walshadow-peerdb")] +struct Cli { + #[arg(long, env = "WALSHADOW_PEERDB_BIND", default_value = "0.0.0.0:8113")] + bind: String, + /// walshadow-control socket to translate onto + #[arg( + long, + env = "WALSHADOW_CONTROL_SOCKET", + default_value = "/run/walshadow-control.sock" + )] + socket: PathBuf, + #[arg(long, default_value = "/var/lib/walshadow-peerdb/state.json")] + state_file: PathBuf, + /// Shared secret for the Authorization header; unauthenticated when unset + #[arg(long, env = "PEERDB_PASSWORD", hide_env_values = true)] + password: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .init(); + let cli = Cli::parse(); + let app = Arc::new(App { + control: ControlClient::new(cli.socket), + store: Store::load(cli.state_file).await?, + password: cli.password.filter(|p| !p.is_empty()), + version: format!("walshadow-peerdb-{}", env!("CARGO_PKG_VERSION")), + stats: Arc::new(walshadow_peerdb::stats::StatsHistory::new()), + }); + + // Sample the daemon's cumulative rows-synced counter on a timer so + // cdc_graph can serve a sync-history series (the shim keeps no history and + // the control socket exposes only the live aggregate). + tokio::spawn({ + let app = app.clone(); + async move { + let mut tick = tokio::time::interval(std::time::Duration::from_secs(15)); + loop { + tick.tick().await; + if let Ok(status) = app.control.call("status", &toml::Table::new()).await { + let rows = walshadow_peerdb::handlers::rows_synced_from_status(&status); + app.stats.record(walshadow_peerdb::pb::now_unix(), rows); + } + } + } + }); + let listener = tokio::net::TcpListener::bind(&cli.bind) + .await + .with_context(|| format!("bind {}", cli.bind))?; + tracing::info!(bind = %cli.bind, "peerdb api shim listening"); + let http = hyper::server::conn::http1::Builder::new(); + let mut shutdown = pin!(shutdown_signal()); + loop { + tokio::select! { + biased; + _ = &mut shutdown => return Ok(()), + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { continue }; + let app = app.clone(); + let conn = http.serve_connection( + TokioIo::new(stream), + service_fn(move |req| { + let app = app.clone(); + async move { Ok::<_, Infallible>(handle(&app, req).await) } + }), + ); + tokio::spawn(async move { + if let Err(e) = conn.await { + tracing::debug!(error = %e, "serve connection"); + } + }); + } + } + } +} + +async fn shutdown_signal() { + let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("install SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = sigterm.recv() => {} + } +} diff --git a/walshadow-peerdb/src/model.rs b/walshadow-peerdb/src/model.rs new file mode 100644 index 0000000..04af5ef --- /dev/null +++ b/walshadow-peerdb/src/model.rs @@ -0,0 +1,710 @@ +//! Hand-written serde structs for the consumed subset of PeerDB's +//! `route.proto`/`peers.proto`/`flow.proto` (grpc-gateway JSON encoding). +//! Deserialization is tolerant: unknown fields ignored, missing fields +//! defaulted, enums accepted as names or numbers, snake_case accepted +//! alongside lowerCamelCase — matches proto3 semantics so PeerDB clients +//! evolve without lockstep shim releases + +use std::collections::HashMap; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; + +use crate::pb::{EnumToken, enum_name_or_number, i64_str}; + +fn flex_u64<'de, D: Deserializer<'de>>(d: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Num(u64), + Str(String), + } + match Raw::deserialize(d)? { + Raw::Num(n) => Ok(n), + Raw::Str(s) => s.parse().map_err(serde::de::Error::custom), + } +} + +fn flex_i64<'de, D: Deserializer<'de>>(d: D) -> Result { + i64_str::deserialize(d) +} + +// ---------------------------------------------------------------- enums + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FlowStatus { + #[default] + Unknown, + Running, + Paused, + Pausing, + Setup, + Snapshot, + Terminating, + Terminated, + Completed, + Resync, + Failed, + Modifying, +} + +impl FlowStatus { + pub fn as_str(self) -> &'static str { + match self { + FlowStatus::Unknown => "STATUS_UNKNOWN", + FlowStatus::Running => "STATUS_RUNNING", + FlowStatus::Paused => "STATUS_PAUSED", + FlowStatus::Pausing => "STATUS_PAUSING", + FlowStatus::Setup => "STATUS_SETUP", + FlowStatus::Snapshot => "STATUS_SNAPSHOT", + FlowStatus::Terminating => "STATUS_TERMINATING", + FlowStatus::Terminated => "STATUS_TERMINATED", + FlowStatus::Completed => "STATUS_COMPLETED", + FlowStatus::Resync => "STATUS_RESYNC", + FlowStatus::Failed => "STATUS_FAILED", + FlowStatus::Modifying => "STATUS_MODIFYING", + } + } + + fn from_token(t: &EnumToken) -> Self { + let all = [ + FlowStatus::Unknown, + FlowStatus::Running, + FlowStatus::Paused, + FlowStatus::Pausing, + FlowStatus::Setup, + FlowStatus::Snapshot, + FlowStatus::Terminating, + FlowStatus::Terminated, + FlowStatus::Completed, + FlowStatus::Resync, + FlowStatus::Failed, + FlowStatus::Modifying, + ]; + match t { + EnumToken::Name(s) => all + .into_iter() + .find(|v| v.as_str() == s) + .unwrap_or_default(), + EnumToken::Number(n) => usize::try_from(*n) + .ok() + .and_then(|i| all.get(i).copied()) + .unwrap_or_default(), + } + } +} + +impl Serialize for FlowStatus { + fn serialize(&self, s: S) -> Result { + s.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for FlowStatus { + fn deserialize>(d: D) -> Result { + Ok(enum_name_or_number(d)? + .map(|t| FlowStatus::from_token(&t)) + .unwrap_or_default()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DbType { + Postgres, + Clickhouse, + Other(String), +} + +impl Default for DbType { + fn default() -> Self { + DbType::Other("UNSPECIFIED".into()) + } +} + +impl DbType { + pub fn as_str(&self) -> &str { + match self { + DbType::Postgres => "POSTGRES", + DbType::Clickhouse => "CLICKHOUSE", + DbType::Other(s) => s, + } + } +} + +impl<'de> Deserialize<'de> for DbType { + fn deserialize>(d: D) -> Result { + Ok(match enum_name_or_number(d)? { + None => DbType::default(), + Some(EnumToken::Name(s)) => match s.as_str() { + "POSTGRES" => DbType::Postgres, + "CLICKHOUSE" => DbType::Clickhouse, + _ => DbType::Other(s), + }, + Some(EnumToken::Number(3)) => DbType::Postgres, + Some(EnumToken::Number(8)) => DbType::Clickhouse, + Some(EnumToken::Number(n)) => DbType::Other(n.to_string()), + }) + } +} + +// -------------------------------------------------------- peer requests + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct PostgresConfig { + pub host: String, + pub port: u32, + pub user: String, + pub password: String, + pub database: String, + #[serde(alias = "require_tls")] + pub require_tls: bool, + #[serde(alias = "disable_tls")] + pub disable_tls: Option, + #[serde(alias = "tls_host")] + pub tls_host: String, + #[serde(alias = "metadata_schema")] + pub metadata_schema: Option, + #[serde(alias = "ssh_config")] + pub ssh_config: Option, + #[serde(alias = "root_ca")] + pub root_ca: Option, + #[serde(alias = "auth_type")] + pub auth_type: Option, + #[serde(alias = "aws_auth")] + pub aws_auth: Option, + #[serde(alias = "skip_cert_verification")] + pub skip_cert_verification: bool, +} + +impl PostgresConfig { + /// disable_tls / require_tls fold into libpq sslmode; walshadow-control + /// takes sslmode verbatim + pub fn sslmode(&self) -> &'static str { + if self.disable_tls == Some(true) { + "disable" + } else if self.require_tls { + "require" + } else { + "prefer" + } + } +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ClickhouseConfig { + pub host: String, + pub port: u32, + pub user: String, + pub password: String, + pub database: String, + #[serde(alias = "s3_path")] + pub s3_path: String, + #[serde(alias = "access_key_id")] + pub access_key_id: String, + #[serde(alias = "secret_access_key")] + pub secret_access_key: String, + pub region: String, + #[serde(alias = "disable_tls")] + pub disable_tls: bool, + pub endpoint: Option, + pub certificate: Option, + #[serde(alias = "private_key")] + pub private_key: Option, + #[serde(alias = "root_ca")] + pub root_ca: Option, + #[serde(alias = "tls_host")] + pub tls_host: String, + pub s3: Option, + pub cluster: String, + pub replicated: bool, + #[serde(alias = "tls_certificate_directory")] + pub tls_certificate_directory: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Peer { + pub name: String, + #[serde(rename = "type")] + pub db_type: DbType, + #[serde(alias = "postgres_config")] + pub postgres_config: Option, + #[serde(alias = "clickhouse_config")] + pub clickhouse_config: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CreatePeerRequest { + pub peer: Option, + #[serde(alias = "allow_update")] + pub allow_update: bool, + #[serde(alias = "disable_validation")] + pub disable_validation: bool, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct ValidatePeerRequest { + pub peer: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct DropPeerRequest { + #[serde(alias = "peer_name")] + pub peer_name: String, +} + +// ------------------------------------------------------- flow requests + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TableMapping { + #[serde(alias = "source_table_identifier")] + pub source_table_identifier: String, + #[serde(alias = "destination_table_identifier")] + pub destination_table_identifier: String, + #[serde(alias = "partition_key")] + pub partition_key: String, + pub exclude: Vec, + pub columns: Vec, + #[serde(deserialize_with = "enum_name_or_number")] + pub engine: Option, + #[serde(alias = "sharding_key")] + pub sharding_key: String, + #[serde(alias = "policy_name")] + pub policy_name: String, + #[serde(alias = "partition_by_expr")] + pub partition_by_expr: String, +} + +impl TableMapping { + /// TableEngine 0 = CH_ENGINE_REPLACING_MERGE_TREE, walshadow's native + /// destination shape; anything else is a divergence worth a WARN + pub fn engine_is_default(&self) -> bool { + match &self.engine { + None => true, + Some(EnumToken::Number(n)) => *n == 0, + Some(EnumToken::Name(s)) => s == "CH_ENGINE_REPLACING_MERGE_TREE", + } + } +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FlowConnectionConfigs { + #[serde(alias = "flow_job_name")] + pub flow_job_name: String, + #[serde(alias = "table_mappings")] + pub table_mappings: Vec, + #[serde(alias = "max_batch_size")] + pub max_batch_size: u32, + #[serde(alias = "idle_timeout_seconds", deserialize_with = "flex_u64")] + pub idle_timeout_seconds: u64, + #[serde(alias = "cdc_staging_path")] + pub cdc_staging_path: String, + #[serde(alias = "publication_name")] + pub publication_name: String, + #[serde(alias = "replication_slot_name")] + pub replication_slot_name: String, + #[serde(alias = "do_initial_snapshot")] + pub do_initial_snapshot: bool, + #[serde(alias = "snapshot_num_rows_per_partition")] + pub snapshot_num_rows_per_partition: u32, + #[serde(alias = "snapshot_num_partitions_override")] + pub snapshot_num_partitions_override: u32, + #[serde(alias = "snapshot_staging_path")] + pub snapshot_staging_path: String, + #[serde(alias = "snapshot_max_parallel_workers")] + pub snapshot_max_parallel_workers: u32, + #[serde(alias = "snapshot_num_tables_in_parallel")] + pub snapshot_num_tables_in_parallel: u32, + pub resync: bool, + #[serde(alias = "initial_snapshot_only")] + pub initial_snapshot_only: bool, + #[serde(alias = "soft_delete_col_name")] + pub soft_delete_col_name: String, + #[serde(alias = "synced_at_col_name")] + pub synced_at_col_name: String, + pub script: String, + #[serde(deserialize_with = "enum_name_or_number")] + pub system: Option, + #[serde(alias = "source_name")] + pub source_name: String, + #[serde(alias = "destination_name")] + pub destination_name: String, + pub env: HashMap, + pub version: u32, + pub flags: Vec, + #[serde(alias = "skip_validation")] + pub skip_validation: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CreateCDCFlowRequest { + #[serde(alias = "connection_configs")] + pub connection_configs: Option, + #[serde(alias = "attach_to_existing")] + pub attach_to_existing: bool, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CdcFlowConfigUpdate { + #[serde(alias = "additional_tables")] + pub additional_tables: Vec, + #[serde(alias = "removed_tables")] + pub removed_tables: Vec, + #[serde(alias = "batch_size")] + pub batch_size: u32, + #[serde(alias = "idle_timeout", deserialize_with = "flex_u64")] + pub idle_timeout: u64, + #[serde(alias = "number_of_syncs")] + pub number_of_syncs: i32, + #[serde(alias = "updated_env")] + pub updated_env: HashMap, + #[serde(alias = "snapshot_num_rows_per_partition")] + pub snapshot_num_rows_per_partition: u32, + #[serde(alias = "snapshot_num_partitions_override")] + pub snapshot_num_partitions_override: u32, + #[serde(alias = "snapshot_max_parallel_workers")] + pub snapshot_max_parallel_workers: u32, + #[serde(alias = "snapshot_num_tables_in_parallel")] + pub snapshot_num_tables_in_parallel: u32, + #[serde(alias = "skip_initial_snapshot_for_table_additions")] + pub skip_initial_snapshot_for_table_additions: bool, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FlowConfigUpdate { + #[serde(alias = "cdc_flow_config_update")] + pub cdc_flow_config_update: Option, + #[serde(alias = "qrep_flow_config_update")] + pub qrep_flow_config_update: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FlowStateChangeRequest { + #[serde(alias = "flow_job_name")] + pub flow_job_name: String, + #[serde(alias = "requested_flow_state")] + pub requested_flow_state: FlowStatus, + #[serde(alias = "flow_config_update")] + pub flow_config_update: Option, + #[serde(alias = "drop_mirror_stats")] + pub drop_mirror_stats: bool, + #[serde(alias = "skip_destination_drop")] + pub skip_destination_drop: bool, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct MirrorStatusRequest { + #[serde(alias = "flow_job_name")] + pub flow_job_name: String, + #[serde(alias = "include_flow_info")] + pub include_flow_info: bool, + #[serde(alias = "exclude_batches")] + pub exclude_batches: bool, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct GetCDCBatchesRequest { + #[serde(alias = "flow_job_name")] + pub flow_job_name: String, + pub limit: u32, + pub ascending: bool, + #[serde(alias = "before_id", deserialize_with = "flex_i64")] + pub before_id: i64, + #[serde(alias = "after_id", deserialize_with = "flex_i64")] + pub after_id: i64, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct GraphRequest { + #[serde(alias = "flow_job_name")] + pub flow_job_name: String, + /// PeerDB TimeAggregateType; accepts the enum number or name + #[serde(alias = "aggregate_type", deserialize_with = "enum_name_or_number")] + pub aggregate_type: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ListMirrorLogsRequest { + #[serde(alias = "flow_job_name")] + pub flow_job_name: String, + pub level: String, + pub page: i32, + #[serde(alias = "num_per_page")] + pub num_per_page: i32, +} + +// -------------------------------------------------------- query params + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct PeerActivityQuery { + #[serde(alias = "peerName")] + pub peer_name: String, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct SchemaTablesQuery { + #[serde(alias = "peerName")] + pub peer_name: String, + #[serde(alias = "schemaName")] + pub schema_name: String, + #[serde(alias = "cdcEnabled")] + pub cdc_enabled: Option, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(default)] +pub struct TableColumnsQuery { + #[serde(alias = "peerName")] + pub peer_name: String, + #[serde(alias = "schemaName")] + pub schema_name: String, + #[serde(alias = "tableName")] + pub table_name: String, +} + +// ----------------------------------------------------------- responses + +#[derive(Serialize)] +pub struct CreatePeerResponse { + pub status: &'static str, + pub message: String, +} + +#[derive(Serialize)] +pub struct ValidatePeerResponse { + pub status: &'static str, + pub message: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateCDCFlowResponse { + pub workflow_id: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ListMirrorsItem { + #[serde(with = "i64_str")] + pub id: i64, + pub workflow_id: String, + pub name: String, + pub source_name: String, + pub source_type: &'static str, + pub destination_name: String, + pub destination_type: &'static str, + /// epoch milliseconds; proto double, PeerDB fills `UnixMilli()` + pub created_at: f64, + pub is_cdc: bool, + pub status: FlowStatus, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CDCBatch { + #[serde(with = "i64_str")] + pub start_lsn: i64, + #[serde(with = "i64_str")] + pub end_lsn: i64, + #[serde(with = "i64_str")] + pub num_rows: i64, + pub start_time: String, + pub end_time: String, + #[serde(with = "i64_str")] + pub batch_id: i64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CDCMirrorStatus { + pub config: Value, + pub snapshot_status: Value, + pub cdc_batches: Vec, + pub source_type: &'static str, + pub destination_type: &'static str, + #[serde(with = "i64_str")] + pub rows_synced: i64, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MirrorStatusResponse { + pub flow_job_name: String, + pub cdc_status: CDCMirrorStatus, + pub current_flow_state: FlowStatus, + pub created_at: String, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TableResponse { + pub table_name: String, + pub can_mirror: bool, + pub table_size: String, + pub is_replica_identity_full: bool, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ColumnsItem { + pub name: String, + #[serde(rename = "type")] + pub column_type: String, + pub is_key: bool, + pub qkind: String, + pub is_replica_identity: bool, +} + +/// LSN fields carry proto names like `redo_lSN`, whose protojson name is +/// `redoLSN`; spelled out per field rather than trusting rename_all +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SlotInfo { + pub slot_name: String, + #[serde(rename = "redoLSN")] + pub redo_lsn: String, + #[serde(rename = "restartLSN")] + pub restart_lsn: String, + pub active: bool, + pub lag_in_mb: f32, + #[serde(rename = "confirmedFlushLSN")] + pub confirmed_flush_lsn: String, + pub wal_status: String, +} + +#[derive(Serialize)] +pub struct PeerListItem<'a> { + pub name: &'a str, + #[serde(rename = "type")] + pub db_type: &'a str, +} + +/// Mask `peerdb_redacted` string fields anywhere in a stored peer config; +/// PeerDB masks with literal `********` +pub fn redact(value: &mut Value) { + const REDACTED: &[&str] = &[ + "password", + "rootCa", + "root_ca", + "accessKeyId", + "access_key_id", + "secretAccessKey", + "secret_access_key", + "certificate", + "privateKey", + "private_key", + "subscriptionId", + "subscription_id", + "apiKey", + "api_key", + ]; + match value { + Value::Object(map) => { + for (k, v) in map.iter_mut() { + if REDACTED.contains(&k.as_str()) { + if let Value::String(s) = v + && !s.is_empty() + { + *s = "********".into(); + } + } else { + redact(v); + } + } + } + Value::Array(items) => items.iter_mut().for_each(redact), + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tolerant_decode_defaults_and_unknowns() { + let req: CreateCDCFlowRequest = serde_json::from_str( + r#"{ + "connectionConfigs": { + "flow_job_name": "m1", + "tableMappings": [ + {"sourceTableIdentifier": "public.users", "futureKnob": 7} + ], + "sourceName": "pg", + "destinationName": "ch", + "idleTimeoutSeconds": "60" + }, + "someFutureField": {"nested": true} + }"#, + ) + .unwrap(); + let cfg = req.connection_configs.unwrap(); + assert_eq!(cfg.flow_job_name, "m1"); + assert_eq!(cfg.idle_timeout_seconds, 60); + assert_eq!( + cfg.table_mappings[0].source_table_identifier, + "public.users" + ); + assert!(!cfg.do_initial_snapshot); + assert!(!req.attach_to_existing); + } + + #[test] + fn enums_accept_names_and_numbers() { + let by_name: FlowStatus = serde_json::from_str("\"STATUS_PAUSED\"").unwrap(); + assert_eq!(by_name, FlowStatus::Paused); + let by_number: FlowStatus = serde_json::from_str("2").unwrap(); + assert_eq!(by_number, FlowStatus::Paused); + let unknown: FlowStatus = serde_json::from_str("\"STATUS_FROM_THE_FUTURE\"").unwrap(); + assert_eq!(unknown, FlowStatus::Unknown); + assert_eq!( + serde_json::to_string(&FlowStatus::Running).unwrap(), + "\"STATUS_RUNNING\"" + ); + + let pg: DbType = serde_json::from_str("3").unwrap(); + assert_eq!(pg, DbType::Postgres); + let ch: DbType = serde_json::from_str("\"CLICKHOUSE\"").unwrap(); + assert_eq!(ch, DbType::Clickhouse); + } + + #[test] + fn peer_decode_both_casings() { + let p: Peer = serde_json::from_str( + r#"{"name": "pg", "type": "POSTGRES", + "postgres_config": {"host": "db", "port": 5432, "requireTls": true}}"#, + ) + .unwrap(); + let cfg = p.postgres_config.unwrap(); + assert_eq!(cfg.host, "db"); + assert_eq!(cfg.sslmode(), "require"); + } + + #[test] + fn redact_masks_nested_secrets() { + let mut v = serde_json::json!({ + "postgresConfig": {"host": "db", "password": "hunter2", "rootCa": ""} + }); + redact(&mut v); + assert_eq!(v["postgresConfig"]["password"], "********"); + assert_eq!(v["postgresConfig"]["rootCa"], ""); + assert_eq!(v["postgresConfig"]["host"], "db"); + } +} diff --git a/walshadow-peerdb/src/pb.rs b/walshadow-peerdb/src/pb.rs new file mode 100644 index 0000000..da81a62 --- /dev/null +++ b/walshadow-peerdb/src/pb.rs @@ -0,0 +1,100 @@ +//! proto3-JSON conventions per grpc-gateway: 64-bit ints as strings, +//! enums as strings (ints accepted on input), Timestamp as RFC 3339, +//! absent field = default value + +use serde::{Deserialize, Deserializer, Serializer}; + +/// Serialize i64 as a JSON string; accept string or number on input +pub mod i64_str { + use super::*; + + pub fn serialize(v: &i64, s: S) -> Result { + s.serialize_str(&v.to_string()) + } + + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Num(i64), + Str(String), + } + match Raw::deserialize(d)? { + Raw::Num(n) => Ok(n), + Raw::Str(s) => s.parse().map_err(serde::de::Error::custom), + } + } +} + +/// Accept a proto3 enum encoded as its name or its number +pub fn enum_name_or_number<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum Raw { + Num(i64), + Str(String), + } + Ok(match Option::::deserialize(d)? { + None => None, + Some(Raw::Num(n)) => Some(EnumToken::Number(n)), + Some(Raw::Str(s)) => Some(EnumToken::Name(s)), + }) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EnumToken { + Name(String), + Number(i64), +} + +pub fn timestamp_rfc3339(unix_secs: i64) -> String { + use chrono::{Datelike, Timelike}; + chrono::DateTime::from_timestamp(unix_secs, 0) + .map(|t| { + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", + t.year(), + t.month(), + t.day(), + t.hour(), + t.minute(), + t.second() + ) + }) + .unwrap_or_default() +} + +pub fn now_unix() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Serialize; + + #[derive(Serialize, Deserialize)] + struct WithI64 { + #[serde(with = "i64_str")] + v: i64, + } + + #[test] + fn i64_roundtrip() { + let j = serde_json::to_string(&WithI64 { v: 1 << 60 }).unwrap(); + assert_eq!(j, format!("{{\"v\":\"{}\"}}", 1i64 << 60)); + let from_str: WithI64 = serde_json::from_str(&j).unwrap(); + assert_eq!(from_str.v, 1 << 60); + let from_num: WithI64 = serde_json::from_str("{\"v\":42}").unwrap(); + assert_eq!(from_num.v, 42); + } + + #[test] + fn timestamp_format() { + assert_eq!(timestamp_rfc3339(0), "1970-01-01T00:00:00Z"); + assert_eq!(timestamp_rfc3339(1784131200), "2026-07-15T16:00:00Z"); + } +} diff --git a/walshadow-peerdb/src/response.rs b/walshadow-peerdb/src/response.rs new file mode 100644 index 0000000..79e3482 --- /dev/null +++ b/walshadow-peerdb/src/response.rs @@ -0,0 +1,62 @@ +//! Response construction over hyper types, filling axum's IntoResponse role + +use http_body_util::Full; +use hyper::body::Bytes; +use hyper::header::{CONTENT_TYPE, HeaderValue}; +use hyper::{Response, StatusCode}; +use serde::Serialize; +use serde_json::json; + +use crate::error::GrpcError; + +pub struct Json(pub T); + +pub trait IntoResponse { + fn into_response(self) -> Response>; +} + +fn json_response(status: StatusCode, buf: Vec) -> Response> { + let mut resp = Response::new(Full::from(buf)); + *resp.status_mut() = status; + resp.headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + resp +} + +impl IntoResponse for Response> { + fn into_response(self) -> Response> { + self + } +} + +impl IntoResponse for Json { + fn into_response(self) -> Response> { + match serde_json::to_vec(&self.0) { + Ok(buf) => json_response(StatusCode::OK, buf), + Err(e) => GrpcError::internal(format!("serialize response: {e}")).into_response(), + } + } +} + +impl IntoResponse for GrpcError { + fn into_response(self) -> Response> { + let body = json!({ + "code": self.code as i32, + "message": self.message, + "details": [], + }); + json_response( + self.code.http_status(), + serde_json::to_vec(&body).unwrap_or_default(), + ) + } +} + +impl IntoResponse for Result { + fn into_response(self) -> Response> { + match self { + Ok(r) => r.into_response(), + Err(e) => e.into_response(), + } + } +} diff --git a/walshadow-peerdb/src/routes.rs b/walshadow-peerdb/src/routes.rs new file mode 100644 index 0000000..fee4219 --- /dev/null +++ b/walshadow-peerdb/src/routes.rs @@ -0,0 +1,255 @@ +use std::sync::Arc; + +use http_body_util::{BodyExt, Limited}; +use hyper::body::{Body, Bytes}; +use hyper::{Request, Response, Uri}; +use serde::de::DeserializeOwned; +use serde_json::Value; + +use crate::auth::require_auth; +use crate::control::ControlClient; +use crate::error::GrpcError; +use crate::handlers::{mirrors, misc, peers}; +use crate::response::IntoResponse; +use crate::state::Store; +use crate::stats::StatsHistory; + +pub struct App { + pub control: ControlClient, + pub store: Store, + /// PEERDB_PASSWORD-style shared secret; unauthenticated when None + pub password: Option, + pub version: String, + /// Sampled rows-synced history backing the sync-history graph + pub stats: Arc, +} + +type BoxError = Box; +type Full = http_body_util::Full; + +/// axum's default request body cap carried over +const BODY_LIMIT: usize = 2 * 1024 * 1024; + +pub async fn handle(app: &App, req: Request) -> Response +where + B: Body, + B::Error: Into, +{ + route(app, req).await.into_response() +} + +async fn route(app: &App, req: Request) -> Result, GrpcError> +where + B: Body, + B::Error: Into, +{ + let (parts, body) = req.into_parts(); + require_auth(app.password.as_deref(), &parts.headers)?; + let path = parts.uri.path(); + Ok(match (parts.method.as_str(), path) { + // mapped: drive the control socket + ("POST", "/v1/peers/create") => peers::create_peer(app, json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/peers/validate") => peers::validate_peer(json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/peers/drop") => peers::drop_peer(app, json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/mirrors/cdc/validate") => mirrors::validate_cdc(app, json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/flows/cdc/create") => mirrors::create_cdc(app, json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/mirrors/state_change") => mirrors::state_change(app, json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/mirrors/status") => mirrors::mirror_status(app, json_body(body).await?) + .await + .into_response(), + ("GET", "/v1/mirrors/list") => mirrors::list_mirrors(app).await.into_response(), + ("GET", "/v1/mirrors/names") => mirrors::list_mirror_names(app).await.into_response(), + // served from shim/control state + ("GET", "/v1/peers/list") => peers::list_peers(app).await.into_response(), + ("GET", "/v1/peers/schemas") => peers::schemas(app, query(&parts.uri)?) + .await + .into_response(), + ("GET", "/v1/peers/tables") => peers::tables_in_schema(app, query(&parts.uri)?) + .await + .into_response(), + ("GET", "/v1/peers/tables/all") => peers::all_tables(app, query(&parts.uri)?) + .await + .into_response(), + ("GET", "/v1/peers/columns") => peers::columns(app, query(&parts.uri)?) + .await + .into_response(), + ("GET", "/v1/peers/columns/all_type_conversions") => { + peers::all_type_conversions().await.into_response() + } + ("POST", "/v1/mirrors/cdc/batches") => { + mirrors::cdc_batches_post(app, json_body(body).await?) + .await + .into_response() + } + ("POST", "/v1/mirrors/cdc/graph") => mirrors::cdc_graph(app, json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/mirrors/logs") => mirrors::mirror_logs(json_body(body).await?) + .await + .into_response(), + ("GET", "/v1/version") => misc::version(app).await.into_response(), + ("GET", "/v1/instance/info") => misc::instance_info(app).await.into_response(), + // accept & ignore + ("GET", "/v1/peers/publications") => peers::publications().await.into_response(), + ("POST", "/v1/peers/slots/lag_history") => peers::slot_lag_history().await.into_response(), + ("GET", "/v1/alerts/config") => misc::alert_configs_get().await.into_response(), + ("POST", "/v1/alerts/config") => misc::alert_config_post().await.into_response(), + ("GET", "/v1/dynamic_settings") => misc::dynamic_settings_get().await.into_response(), + ("POST", "/v1/dynamic_settings") => misc::dynamic_setting_post().await.into_response(), + ("POST", "/v1/scripts") => misc::script_post().await.into_response(), + ("POST", "/v1/flows/tags") => misc::flow_tags_post(json_body(body).await?) + .await + .into_response(), + ("POST", "/v1/instance/maintenance") => misc::maintenance_post().await.into_response(), + ("GET", "/v1/instance/maintenance/status") => { + misc::maintenance_status().await.into_response() + } + ("POST", "/v1/instance/maintenance/skip-snapshot-wait") => { + misc::skip_snapshot_wait().await.into_response() + } + ("POST", "/v1/mirrors/sequences/reset") => misc::sequences_reset().await.into_response(), + ("POST", "/v1/flows/cdc/cancel_table_addition") => { + misc::cancel_table_addition(json_body(body).await?) + .await + .into_response() + } + // reject + ("POST", "/v1/flows/qrep/create") => misc::qrep_create().await.into_response(), + (method, path) => { + if method == "GET" + && let Some(peer_name) = param(path, "/v1/peers/info/") + { + peers::peer_info(app, peer_name).await.into_response() + } else if method == "GET" + && let Some(peer_name) = param(path, "/v1/peers/type/") + { + peers::peer_type(app, peer_name).await.into_response() + } else if method == "GET" + && let Some(peer_name) = param(path, "/v1/peers/slots/") + { + peers::slots(app, peer_name).await.into_response() + } else if method == "GET" + && let Some(peer_name) = param(path, "/v1/peers/stats/") + { + peers::stats(app, peer_name).await.into_response() + } else if method == "GET" + && let Some(flow) = param(path, "/v1/mirrors/cdc/batches/") + { + mirrors::cdc_batches_get(app, flow).await.into_response() + } else if method == "GET" + && let Some(flow) = param(path, "/v1/mirrors/cdc/table_total_counts/") + { + mirrors::table_total_counts(app, flow).await.into_response() + } else if method == "GET" + && let Some(flow) = param(path, "/v1/mirrors/total_rows_synced/") + { + mirrors::total_rows_synced(app, flow).await.into_response() + } else if method == "GET" && param(path, "/v1/mirrors/cdc/initial_load/").is_some() { + mirrors::initial_load_summary().await.into_response() + } else if method == "GET" + && let Some(flow_name) = param(path, "/v1/flows/tags/") + { + misc::flow_tags_get(flow_name).await.into_response() + } else if method == "DELETE" && param(path, "/v1/alerts/config/").is_some() { + misc::alert_config_delete().await.into_response() + } else if method == "GET" && param(path, "/v1/scripts/").is_some() { + misc::scripts_get().await.into_response() + } else if method == "DELETE" && param(path, "/v1/scripts/").is_some() { + misc::script_delete().await.into_response() + } else { + misc::unimplemented_fallback(&parts.uri) + .await + .into_response() + } + } + }) +} + +async fn json_body(body: B) -> Result +where + B: Body, + B::Error: Into, +{ + let bytes = Limited::new(body, BODY_LIMIT) + .collect() + .await + .map_err(|e| GrpcError::invalid(format!("malformed request body: {e}")))? + .to_bytes(); + serde_json::from_slice(&bytes) + .map_err(|e| GrpcError::invalid(format!("malformed request body: {e}"))) +} + +fn query(uri: &Uri) -> Result { + serde_urlencoded::from_str(uri.query().unwrap_or("")) + .map_err(|e| GrpcError::invalid(format!("malformed query string: {e}"))) +} + +/// Trailing single-segment path param, percent-decoded (axum Path semantics) +fn param(path: &str, prefix: &str) -> Option { + let rest = path.strip_prefix(prefix)?; + (!rest.is_empty() && !rest.contains('/')).then(|| percent_decode(rest)) +} + +fn percent_decode(s: &str) -> String { + let b = s.as_bytes(); + let hex = |i: usize| { + b.get(i) + .and_then(|&c| char::from(c).to_digit(16)) + .map(|d| d as u8) + }; + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + if b[i] == b'%' + && let (Some(hi), Some(lo)) = (hex(i + 1), hex(i + 2)) + { + out.push(hi << 4 | lo); + i += 3; + } else { + out.push(b[i]); + i += 1; + } + } + String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(&e.into_bytes()).into_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn path_params() { + assert_eq!( + param("/v1/peers/info/pg", "/v1/peers/info/"), + Some("pg".into()) + ); + assert_eq!( + param("/v1/peers/info/a%20b", "/v1/peers/info/"), + Some("a b".into()) + ); + assert_eq!(param("/v1/peers/info/", "/v1/peers/info/"), None); + assert_eq!(param("/v1/peers/info/a/b", "/v1/peers/info/"), None); + assert_eq!(param("/v1/other", "/v1/peers/info/"), None); + } + + #[test] + fn percent_decoding() { + assert_eq!(percent_decode("plain"), "plain"); + assert_eq!(percent_decode("a%2Fb%3f"), "a/b?"); + // stray % passes through + assert_eq!(percent_decode("100%"), "100%"); + assert_eq!(percent_decode("%zz"), "%zz"); + } +} diff --git a/walshadow-peerdb/src/state.rs b/walshadow-peerdb/src/state.rs new file mode 100644 index 0000000..877c9c9 --- /dev/null +++ b/walshadow-peerdb/src/state.rs @@ -0,0 +1,149 @@ +//! Shim-local persistence: peer registry + the single mirror record. +//! Connection-parameter truth lives in walshadow-control's state; this +//! copy exists to echo `GetPeerInfo` and re-derive source/dest role on +//! peer reference. Single writer, same durability model as control's +//! `state.json` + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tokio::sync::Mutex; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Role { + Source, + Dest, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PeerRecord { + /// DBType name, `POSTGRES` / `CLICKHOUSE` + pub db_type: String, + pub role: Role, + /// submitted `*_config` verbatim, echoed (redacted) by GetPeerInfo + pub config: Value, + pub created_at_unix: i64, +} + +/// Dotted strings exist only at control-line interpolation; stored and +/// compared as (namespace, relname) pairs +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TableRef { + pub namespace: String, + pub relname: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MirrorRecord { + pub name: String, + /// echo of `flow_job_name`; no Temporal behind it + pub workflow_id: String, + pub source_name: String, + pub destination_name: String, + /// opt-in set of source tables + pub tables: Vec, + pub do_initial_snapshot: bool, + pub created_at_unix: i64, + /// submitted FlowConnectionConfigs verbatim, echoed by MirrorStatus + pub config: Value, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ShimState { + #[serde(default)] + pub peers: BTreeMap, + #[serde(default)] + pub mirror: Option, + /// names of terminated mirrors; MirrorStatus answers + /// STATUS_TERMINATED for these while ListMirrors stays empty + #[serde(default)] + pub terminated: Vec, +} + +impl ShimState { + pub fn peer_by_role(&self, role: Role) -> Option<(&String, &PeerRecord)> { + self.peers.iter().find(|(_, p)| p.role == role) + } +} + +pub struct Store { + path: PathBuf, + state: Mutex, +} + +impl Store { + pub async fn load(path: PathBuf) -> Result { + let state = match tokio::fs::read(&path).await { + Ok(bytes) => serde_json::from_slice(&bytes) + .with_context(|| format!("parse state file {}", path.display()))?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => ShimState::default(), + Err(e) => return Err(e).with_context(|| format!("read state file {}", path.display())), + }; + Ok(Self { + path, + state: Mutex::new(state), + }) + } + + pub async fn get(&self) -> ShimState { + self.state.lock().await.clone() + } + + /// Mutate under the lock, then persist; closure's return value is + /// passed back to the caller + pub async fn update(&self, f: impl FnOnce(&mut ShimState) -> T) -> Result { + let mut guard = self.state.lock().await; + let out = f(&mut guard); + persist(&self.path, &guard).await?; + Ok(out) + } +} + +async fn persist(path: &Path, state: &ShimState) -> Result<()> { + if let Some(dir) = path.parent() + && !dir.as_os_str().is_empty() + { + tokio::fs::create_dir_all(dir) + .await + .with_context(|| format!("create state dir {}", dir.display()))?; + } + let bytes = serde_json::to_vec_pretty(state).context("serialize state")?; + tokio::fs::write(path, &bytes) + .await + .with_context(|| format!("write state file {}", path.display()))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn roundtrips_state_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.json"); + let store = Store::load(path.clone()).await.unwrap(); + store + .update(|s| { + s.peers.insert( + "pg".into(), + PeerRecord { + db_type: "POSTGRES".into(), + role: Role::Source, + config: serde_json::json!({"host": "db"}), + created_at_unix: 1, + }, + ); + }) + .await + .unwrap(); + let reloaded = Store::load(path).await.unwrap(); + let state = reloaded.get().await; + assert_eq!(state.peers["pg"].db_type, "POSTGRES"); + assert_eq!(state.peer_by_role(Role::Source).unwrap().0, "pg"); + assert!(state.peer_by_role(Role::Dest).is_none()); + } +} diff --git a/walshadow-peerdb/src/stats.rs b/walshadow-peerdb/src/stats.rs new file mode 100644 index 0000000..41bd0fa --- /dev/null +++ b/walshadow-peerdb/src/stats.rs @@ -0,0 +1,98 @@ +//! Rolling samples of the daemon's cumulative `rows_synced` counter. The shim +//! keeps no history of its own and the control socket exposes only a live +//! aggregate, so PeerDB's sync-history graph is synthesized by sampling that +//! counter on a timer and serving per-bucket deltas. + +use std::collections::{BTreeMap, VecDeque}; +use std::sync::Mutex; + +#[derive(Clone, Copy)] +struct Sample { + unix_secs: i64, + rows: i64, +} + +pub struct StatsHistory { + samples: Mutex>, + cap: usize, +} + +impl Default for StatsHistory { + fn default() -> Self { + // 15s cadence * 5760 ~= 24h of retained history + Self { + samples: Mutex::new(VecDeque::new()), + cap: 5760, + } + } +} + +impl StatsHistory { + pub fn new() -> Self { + Self::default() + } + + /// Record one cumulative-counter reading. A drop (the daemon restart resets + /// the counter) clears history so bucket deltas never go negative. + pub fn record(&self, unix_secs: i64, rows: i64) { + let mut s = self.samples.lock().unwrap(); + match s.back() { + Some(last) if rows < last.rows => s.clear(), + Some(last) if last.unix_secs == unix_secs => { + s.pop_back(); + } + _ => {} + } + s.push_back(Sample { unix_secs, rows }); + while s.len() > self.cap { + s.pop_front(); + } + } + + /// Rows synced per `bucket_secs`-wide interval as `(bucket_start_ms, rows)` + /// points, oldest first. Each consecutive-sample delta lands in the bucket + /// its endpoint falls into. + pub fn graph(&self, bucket_secs: i64) -> Vec<(f64, f64)> { + if bucket_secs <= 0 { + return Vec::new(); + } + let s = self.samples.lock().unwrap(); + let mut buckets: BTreeMap = BTreeMap::new(); + let mut prev: Option = None; + for &cur in s.iter() { + if let Some(p) = prev { + let delta = (cur.rows - p.rows).max(0); + let bucket = (cur.unix_secs / bucket_secs) * bucket_secs; + *buckets.entry(bucket).or_default() += delta; + } + prev = Some(cur); + } + buckets + .into_iter() + .map(|(bucket, rows)| ((bucket * 1000) as f64, rows as f64)) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn buckets_deltas_and_resets() { + let h = StatsHistory::new(); + // three 5-min buckets (bucket_secs=300) + h.record(0, 0); + h.record(150, 100); // bucket 0: +100 + h.record(300, 250); // bucket 300: +150 + h.record(450, 250); // bucket 300: +0 + let g = h.graph(300); + assert_eq!(g, vec![(0.0, 100.0), (300_000.0, 150.0)]); + + // counter reset drops history; a lone sample yields no deltas + h.record(600, 5); + assert!(h.graph(300).is_empty()); + h.record(660, 55); + assert_eq!(h.graph(300), vec![(600_000.0, 50.0)]); + } +} diff --git a/walshadow-peerdb/src/warn.rs b/walshadow-peerdb/src/warn.rs new file mode 100644 index 0000000..622e31b --- /dev/null +++ b/walshadow-peerdb/src/warn.rs @@ -0,0 +1,14 @@ +//! Accepted-but-ignored request fields log WARN once per key so silent +//! divergence from PeerDB semantics stays greppable without flooding + +use std::collections::HashSet; +use std::sync::{Mutex, OnceLock}; + +static SEEN: OnceLock>> = OnceLock::new(); + +pub fn warn_ignored(key: &str, detail: &str) { + let seen = SEEN.get_or_init(|| Mutex::new(HashSet::new())); + if seen.lock().is_ok_and(|mut s| s.insert(key.to_string())) { + tracing::warn!(field = key, detail, "ignoring unsupported PeerDB field"); + } +} diff --git a/walshadow-peerdb/tests/http.rs b/walshadow-peerdb/tests/http.rs new file mode 100644 index 0000000..68b05d2 --- /dev/null +++ b/walshadow-peerdb/tests/http.rs @@ -0,0 +1,739 @@ +//! Acceptance drills from plans/future/peerdb.md against an in-process +//! mock control daemon speaking the TOML socket protocol + +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use http_body_util::{BodyExt, Full}; +use hyper::body::Bytes; +use hyper::{Request, StatusCode}; +use serde_json::{Value, json}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use toml::{Table, Value as Toml}; + +use walshadow_peerdb::control::ControlClient; +use walshadow_peerdb::routes::{App, handle}; +use walshadow_peerdb::state::Store; + +/// Fixed source catalog: (namespace, relname, relreplident) +const CATALOG: &[(&str, &str, &str)] = &[ + ("public", "users", "d"), + ("public", "orders", "f"), + ("audit", "log", "d"), +]; + +/// Mock daemon mirroring the real one: `apply` merges into an accumulated +/// config table, `unset` masks it, and the read verbs answer from that +/// state so the handlers exercise real config-fragment logic +struct MockControl { + cfg: Arc>, + rows_synced: Arc>, +} + +fn merge(base: &mut Table, over: Table) { + for (k, v) in over { + match (base.get_mut(&k), v) { + (Some(Toml::Table(bt)), Toml::Table(ot)) => merge(bt, ot), + (_, v) => { + base.insert(k, v); + } + } + } +} + +fn mask(root: &mut Table, m: &Table) { + for (k, v) in m { + if let Toml::Table(sub) = v { + if let Some(Toml::Table(t)) = root.get_mut(k) { + mask(t, sub); + } + } else { + root.remove(k); + } + } +} + +fn selected_set(cfg: &Table) -> HashSet<(String, String)> { + let mut out = HashSet::new(); + if let Some(Toml::Table(tbl)) = cfg.get("table") { + for (ns, nsv) in tbl { + if let Some(nst) = nsv.as_table() { + for rel in nst.keys() { + out.insert((ns.clone(), rel.clone())); + } + } + } + } + out +} + +impl MockControl { + fn spawn(dir: &std::path::Path) -> (PathBuf, Arc) { + let socket = dir.join("control.sock"); + let mock = Arc::new(MockControl { + cfg: Arc::new(Mutex::new(Table::new())), + rows_synced: Arc::new(Mutex::new(0)), + }); + let listener = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + listener.set_nonblocking(true).unwrap(); + let listener = tokio::net::UnixListener::from_std(listener).unwrap(); + let m = mock.clone(); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let m = m.clone(); + tokio::spawn(async move { + // read to EOF: the client half-closes after writing + let mut buf = Vec::new(); + let mut chunk = [0u8; 512]; + loop { + match stream.read(&mut chunk).await { + Ok(0) => break, + Ok(n) => buf.extend_from_slice(&chunk[..n]), + Err(_) => return, + } + } + let text = String::from_utf8_lossy(&buf); + let (verb, body) = text.split_once('\n').unwrap_or((text.as_ref(), "")); + let body: Table = if body.trim().is_empty() { + Table::new() + } else { + body.parse().unwrap() + }; + let resp = m.handle(verb.trim(), body); + let _ = stream.write_all(resp.as_bytes()).await; + let _ = stream.shutdown().await; + }); + } + }); + (socket, mock) + } + + fn handle(&self, verb: &str, body: Table) -> String { + match verb { + "apply" => { + merge(&mut self.cfg.lock().unwrap(), body); + "OK\n".into() + } + "unset" => { + mask(&mut self.cfg.lock().unwrap(), &body); + "OK\n".into() + } + "reload" => "OK\n".into(), + "status" => ok(&self.status_table()), + "tables" => ok(&self.tables_table(body.get("namespace").and_then(Toml::as_str))), + "schemas" => { + let mut ns: Vec<&str> = CATALOG.iter().map(|(n, _, _)| *n).collect(); + ns.sort(); + ns.dedup(); + let mut out = Table::new(); + out.insert( + "schemas".into(), + Toml::Array(ns.into_iter().map(Into::into).collect()), + ); + ok(&out) + } + "columns" => ok(&columns_table(body.get("relname").and_then(Toml::as_str))), + other => format!("ERR unknown command {other}\n"), + } + } + + fn status_table(&self) -> Table { + let cfg = self.cfg.lock().unwrap(); + let paused = cfg + .get("stream") + .and_then(Toml::as_table) + .and_then(|t| t.get("paused")) + .and_then(Toml::as_bool) + .unwrap_or(false); + let mut t = Table::new(); + t.insert("paused".into(), paused.into()); + t.insert( + "rows_synced".into(), + (*self.rows_synced.lock().unwrap()).into(), + ); + t.insert("backfills_pending".into(), 0i64.into()); + t.insert("lag_bytes".into(), 2_097_152i64.into()); + t.insert("lag_seconds".into(), 0.0.into()); + t.insert("uptime_secs".into(), 0i64.into()); + t + } + + fn tables_table(&self, ns_filter: Option<&str>) -> Table { + let selected = selected_set(&self.cfg.lock().unwrap()); + let mut arr = Vec::new(); + for (ns, name, ri) in CATALOG { + if ns_filter.is_some_and(|f| f != *ns) { + continue; + } + let mut t = Table::new(); + t.insert( + "selected".into(), + selected + .contains(&(ns.to_string(), name.to_string())) + .into(), + ); + t.insert("replica_identity".into(), (*ri).into()); + t.insert("namespace".into(), (*ns).into()); + t.insert("name".into(), (*name).into()); + arr.push(Toml::Table(t)); + } + let mut out = Table::new(); + out.insert("tables".into(), Toml::Array(arr)); + out + } + + fn cfg(&self) -> Table { + self.cfg.lock().unwrap().clone() + } +} + +fn ok(body: &Table) -> String { + format!("OK\n{}", toml::to_string(body).unwrap()) +} + +fn columns_table(relname: Option<&str>) -> Table { + let cols: &[(&str, &str, bool)] = match relname { + Some("users") => &[("id", "bigint", true), ("email", "text", false)], + _ => &[], + }; + let arr = cols + .iter() + .map(|(n, ty, nn)| { + let mut t = Table::new(); + t.insert("name".into(), (*n).into()); + t.insert("type".into(), (*ty).into()); + t.insert("notnull".into(), (*nn).into()); + Toml::Table(t) + }) + .collect(); + let mut out = Table::new(); + out.insert("columns".into(), Toml::Array(arr)); + out +} + +/// Read a scalar at `cfg[section][key]` for assertions +fn at<'a>(cfg: &'a Table, section: &str, key: &str) -> Option<&'a Toml> { + cfg.get(section) + .and_then(Toml::as_table) + .and_then(|t| t.get(key)) +} + +/// Whether the accumulated config opts `ns.rel` in +fn opted_in(cfg: &Table, ns: &str, rel: &str) -> bool { + cfg.get("table") + .and_then(Toml::as_table) + .and_then(|t| t.get(ns)) + .and_then(Toml::as_table) + .is_some_and(|t| t.contains_key(rel)) +} + +async fn shim(dir: &std::path::Path, password: Option<&str>) -> (App, Arc) { + let (socket, mock) = MockControl::spawn(dir); + let app = App { + control: ControlClient::new(socket), + store: Store::load(dir.join("state.json")).await.unwrap(), + password: password.map(str::to_string), + version: "walshadow-peerdb-test".into(), + stats: Arc::new(walshadow_peerdb::stats::StatsHistory::new()), + }; + (app, mock) +} + +async fn call(app: &App, method: &str, path: &str, body: Option) -> (StatusCode, Value) { + let req = Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(match &body { + Some(v) => Full::new(Bytes::from(v.to_string())), + None => Full::default(), + }) + .unwrap(); + let resp = handle(app, req).await; + let status = resp.status(); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let value = if bytes.is_empty() { + Value::Null + } else { + serde_json::from_slice(&bytes).unwrap_or(Value::Null) + }; + (status, value) +} + +fn pg_peer(name: &str) -> Value { + json!({"peer": { + "name": name, "type": "POSTGRES", + "postgresConfig": { + "host": "src-db", "port": 5432, "user": "postgres", + "password": "pgpw", "database": "app" + } + }}) +} + +fn ch_peer(name: &str) -> Value { + json!({"peer": { + "name": name, "type": "CLICKHOUSE", + "clickhouseConfig": { + "host": "ch", "port": 9000, "user": "default", + "password": "chpw", "database": "cdc", "disableTls": true + } + }}) +} + +fn cdc_create(flow: &str, tables: &[&str]) -> Value { + json!({"connectionConfigs": { + "flowJobName": flow, + "sourceName": "pg", "destinationName": "ch", + "doInitialSnapshot": true, + "tableMappings": tables + .iter() + .map(|t| json!({"sourceTableIdentifier": t})) + .collect::>(), + }}) +} + +#[tokio::test] +async fn curl_lifecycle() { + let dir = tempfile::tempdir().unwrap(); + let (app, mock) = shim(dir.path(), None).await; + + // create both peers + let (status, body) = call(&app, "POST", "/v1/peers/create", Some(pg_peer("pg"))).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["status"], "CREATED"); + let (status, body) = call(&app, "POST", "/v1/peers/create", Some(ch_peer("ch"))).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["status"], "CREATED"); + let cfg = mock.cfg(); + assert_eq!(at(&cfg, "source", "host").unwrap().as_str(), Some("src-db")); + assert_eq!(at(&cfg, "source", "port").unwrap().as_integer(), Some(5432)); + assert_eq!(at(&cfg, "source", "dbname").unwrap().as_str(), Some("app")); + assert_eq!( + at(&cfg, "source", "sslmode").unwrap().as_str(), + Some("prefer") + ); + assert_eq!(at(&cfg, "ch", "host").unwrap().as_str(), Some("ch")); + assert_eq!(at(&cfg, "ch", "port").unwrap().as_integer(), Some(9000)); + assert_eq!(at(&cfg, "ch", "database").unwrap().as_str(), Some("cdc")); + assert_eq!(at(&cfg, "ch", "secure").unwrap().as_bool(), Some(false)); + + // validate both (structural under the TOML protocol) + let (status, body) = call(&app, "POST", "/v1/peers/validate", Some(pg_peer("pg"))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["status"], "VALID"); + let (_, body) = call(&app, "POST", "/v1/peers/validate", Some(ch_peer("ch"))).await; + assert_eq!(body["status"], "VALID"); + + // validate then create the mirror over two tables + let create = cdc_create("m1", &["public.users", "public.orders"]); + let (status, body) = call( + &app, + "POST", + "/v1/mirrors/cdc/validate", + Some(create.clone()), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + let (status, body) = call(&app, "POST", "/v1/flows/cdc/create", Some(create.clone())).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["workflowId"], "m1"); + let cfg = mock.cfg(); + assert!( + opted_in(&cfg, "public", "users") && opted_in(&cfg, "public", "orders"), + "{cfg:?}" + ); + assert_eq!(at(&cfg, "stream", "paused").unwrap().as_bool(), Some(false)); + + // duplicate create without attach → ALREADY_EXISTS + let (status, body) = call(&app, "POST", "/v1/flows/cdc/create", Some(create.clone())).await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(body["code"], 6); + // with attach → recorded workflow id + let mut attach = create.clone(); + attach["attachToExisting"] = json!(true); + let (status, body) = call(&app, "POST", "/v1/flows/cdc/create", Some(attach)).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["workflowId"], "m1"); + + // status: running, rows synced from the status reply + *mock.rows_synced.lock().unwrap() = 1234; + let (status, body) = call( + &app, + "POST", + "/v1/mirrors/status", + Some(json!({"flowJobName": "m1"})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["currentFlowState"], "STATUS_RUNNING"); + assert_eq!(body["cdcStatus"]["rowsSynced"], "1234"); + assert_eq!(body["cdcStatus"]["config"]["flowJobName"], "m1"); + assert_eq!(body["cdcStatus"]["cdcBatches"][0]["numRows"], "1234"); + + // rows counter endpoints + let (_, body) = call(&app, "GET", "/v1/mirrors/total_rows_synced/m1", None).await; + assert_eq!(body["totalCount"], "1234"); + + // pause → stream.paused = true, status PAUSED + let (status, _) = call( + &app, + "POST", + "/v1/mirrors/state_change", + Some(json!({"flowJobName": "m1", "requestedFlowState": "STATUS_PAUSED"})), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + at(&mock.cfg(), "stream", "paused").unwrap().as_bool(), + Some(true) + ); + let (_, body) = call( + &app, + "POST", + "/v1/mirrors/status", + Some(json!({"flowJobName": "m1"})), + ) + .await; + assert_eq!(body["currentFlowState"], "STATUS_PAUSED"); + + // resume + let (status, _) = call( + &app, + "POST", + "/v1/mirrors/state_change", + Some(json!({"flowJobName": "m1", "requestedFlowState": "STATUS_RUNNING"})), + ) + .await; + assert_eq!(status, StatusCode::OK); + + // additionalTables grows the opt-in set + let (status, body) = call( + &app, + "POST", + "/v1/mirrors/state_change", + Some(json!({ + "flowJobName": "m1", + "requestedFlowState": "STATUS_UNKNOWN", + "flowConfigUpdate": {"cdcFlowConfigUpdate": { + "additionalTables": [{"sourceTableIdentifier": "audit.log"}] + }}, + })), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + let cfg = mock.cfg(); + assert!( + opted_in(&cfg, "public", "users") + && opted_in(&cfg, "public", "orders") + && opted_in(&cfg, "audit", "log"), + "{cfg:?}" + ); + + // mirror listing shows the singleton + let (_, body) = call(&app, "GET", "/v1/mirrors/list", None).await; + assert_eq!(body["mirrors"][0]["name"], "m1"); + assert_eq!(body["mirrors"][0]["status"], "STATUS_RUNNING"); + let (_, body) = call(&app, "GET", "/v1/mirrors/names", None).await; + assert_eq!(body["names"], json!(["m1"])); + + // terminate stops, clears, forgets; list empties, status answers TERMINATED + let (status, _) = call( + &app, + "POST", + "/v1/mirrors/state_change", + Some(json!({"flowJobName": "m1", "requestedFlowState": "STATUS_TERMINATED"})), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!( + !mock.cfg().contains_key("table"), + "terminate clears the opt-in set" + ); + let (_, body) = call(&app, "GET", "/v1/mirrors/list", None).await; + assert_eq!(body["mirrors"], json!([])); + let (status, body) = call( + &app, + "POST", + "/v1/mirrors/status", + Some(json!({"flowJobName": "m1"})), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["currentFlowState"], "STATUS_TERMINATED"); +} + +#[tokio::test] +async fn peer_registry_rules() { + let dir = tempfile::tempdir().unwrap(); + let (app, _mock) = shim(dir.path(), None).await; + + let (status, _) = call(&app, "POST", "/v1/peers/create", Some(pg_peer("pg"))).await; + assert_eq!(status, StatusCode::OK); + // same name again without allowUpdate → 409 + let (status, body) = call(&app, "POST", "/v1/peers/create", Some(pg_peer("pg"))).await; + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(body["code"], 6); + // same name with allowUpdate → ok + let mut update = pg_peer("pg"); + update["allowUpdate"] = json!(true); + let (status, _) = call(&app, "POST", "/v1/peers/create", Some(update)).await; + assert_eq!(status, StatusCode::OK); + // second postgres peer under a different name → FAILED, slot held + let (status, body) = call(&app, "POST", "/v1/peers/create", Some(pg_peer("pg2"))).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["status"], "FAILED"); + + // info is redacted, type echoes + let (_, body) = call(&app, "GET", "/v1/peers/info/pg", None).await; + assert_eq!(body["peer"]["postgresConfig"]["password"], "********"); + assert_eq!(body["peer"]["postgresConfig"]["host"], "src-db"); + let (_, body) = call(&app, "GET", "/v1/peers/type/pg", None).await; + assert_eq!(body["peerType"], "POSTGRES"); + let (status, body) = call(&app, "GET", "/v1/peers/info/nope", None).await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(body["code"], 5); + + // list buckets by role + let (_, body) = call(&app, "POST", "/v1/peers/create", Some(ch_peer("ch"))).await; + assert_eq!(body["status"], "CREATED"); + let (_, body) = call(&app, "GET", "/v1/peers/list", None).await; + assert_eq!(body["items"].as_array().unwrap().len(), 2); + assert_eq!(body["sourceItems"][0]["name"], "pg"); + assert_eq!(body["destinationItems"][0]["name"], "ch"); + + // drop refused while mirror references the peer + let (status, _) = call( + &app, + "POST", + "/v1/flows/cdc/create", + Some(cdc_create("m1", &["public.users"])), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (status, body) = call( + &app, + "POST", + "/v1/peers/drop", + Some(json!({"peerName": "pg"})), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["code"], 9); +} + +#[tokio::test] +async fn introspection_endpoints() { + let dir = tempfile::tempdir().unwrap(); + let (app, _mock) = shim(dir.path(), None).await; + call(&app, "POST", "/v1/peers/create", Some(pg_peer("pg"))).await; + + // schemas comes straight from the `schemas` verb + let (status, body) = call(&app, "GET", "/v1/peers/schemas?peer_name=pg", None).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["schemas"], json!(["audit", "public"])); + + let (_, body) = call( + &app, + "GET", + "/v1/peers/tables?peerName=pg&schemaName=public", + None, + ) + .await; + let tables = body["tables"].as_array().unwrap(); + assert_eq!(tables.len(), 2); + assert_eq!(tables[0]["tableName"], "users"); + assert_eq!(tables[1]["isReplicaIdentityFull"], true); + + let (_, body) = call(&app, "GET", "/v1/peers/tables/all?peer_name=pg", None).await; + assert_eq!( + body["tables"], + json!(["public.users", "public.orders", "audit.log"]) + ); + + // columns answers from the `columns` verb + let (status, body) = call( + &app, + "GET", + "/v1/peers/columns?peer_name=pg&schema_name=public&table_name=users", + None, + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + let cols = body["columns"].as_array().unwrap(); + assert_eq!(cols.len(), 2); + assert_eq!(cols[0]["name"], "id"); + assert_eq!(cols[0]["type"], "bigint"); + + // slots synthesized from status; the daemon streams unless paused, so an + // unpaused config presents the slot as active + let (status, body) = call(&app, "GET", "/v1/peers/slots/pg", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["slotData"][0]["slotName"], "walshadow"); + assert_eq!(body["slotData"][0]["active"], true); + let (_, body) = call(&app, "GET", "/v1/peers/stats/pg", None).await; + assert_eq!(body["statData"], json!([])); +} + +#[tokio::test] +async fn ignore_and_reject_surface() { + let dir = tempfile::tempdir().unwrap(); + let (app, _mock) = shim(dir.path(), None).await; + call(&app, "POST", "/v1/peers/create", Some(pg_peer("pg"))).await; + call(&app, "POST", "/v1/peers/create", Some(ch_peer("ch"))).await; + + // alerts config accepts with a success shape + let (status, _) = call( + &app, + "POST", + "/v1/alerts/config", + Some(json!({"config": {"serviceType": "slack"}})), + ) + .await; + assert_eq!(status, StatusCode::OK); + let (_, body) = call(&app, "GET", "/v1/alerts/config", None).await; + assert_eq!(body["configs"], json!([])); + + // publications are empty by model + let (_, body) = call(&app, "GET", "/v1/peers/publications?peer_name=pg", None).await; + assert_eq!(body["publicationNames"], json!([])); + + // qrep create → 501 with grpc-shaped body + let (status, body) = call( + &app, + "POST", + "/v1/flows/qrep/create", + Some(json!({"qrepConfig": {"flowJobName": "q"}})), + ) + .await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + assert_eq!(body["code"], 12); + assert!(body["message"].as_str().unwrap().contains("qrep")); + + // create carrying ignored fields succeeds (WARN once, not observable here) + let mut create = cdc_create("m1", &["public.users"]); + create["connectionConfigs"]["softDeleteColName"] = json!("_peerdb_is_deleted"); + create["connectionConfigs"]["publicationName"] = json!("pub"); + let (status, body) = call(&app, "POST", "/v1/flows/cdc/create", Some(create)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + // initialSnapshotOnly / resync are honest rejections + let mut snap = cdc_create("m2", &["public.users"]); + snap["connectionConfigs"]["initialSnapshotOnly"] = json!(true); + let (status, _) = call(&app, "POST", "/v1/mirrors/cdc/validate", Some(snap)).await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + + // per-table rename rejected until runtime-config rename exists + let mut renamed = cdc_create("m3", &["public.users"]); + renamed["connectionConfigs"]["tableMappings"][0]["destinationTableIdentifier"] = + json!("renamed"); + let (status, _) = call(&app, "POST", "/v1/mirrors/cdc/validate", Some(renamed)).await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + + // unknown /v1 path → 501 grpc shape + let (status, body) = call(&app, "GET", "/v1/flows/unheard_of", None).await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + assert_eq!(body["code"], 12); + + // instance/version render + let (_, body) = call(&app, "GET", "/v1/version", None).await; + assert_eq!(body["version"], "walshadow-peerdb-test"); + let (_, body) = call(&app, "GET", "/v1/instance/info", None).await; + assert_eq!(body["status"], "INSTANCE_STATUS_READY"); +} + +#[tokio::test] +async fn tolerant_decode_and_errors() { + let dir = tempfile::tempdir().unwrap(); + let (app, mock) = shim(dir.path(), None).await; + call(&app, "POST", "/v1/peers/create", Some(pg_peer("pg"))).await; + call(&app, "POST", "/v1/peers/create", Some(ch_peer("ch"))).await; + + // fields from a newer PeerDB release parse and apply + let mut create = cdc_create("m1", &["public.users"]); + create["connectionConfigs"]["fieldFromTheFuture"] = json!({"nested": [1, 2]}); + create["unknownTopLevel"] = json!("x"); + let (status, body) = call(&app, "POST", "/v1/flows/cdc/create", Some(create)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + + // TOML bodies carry spaces: a spaced password applies verbatim + let mut spaced = pg_peer("pg"); + spaced["peer"]["postgresConfig"]["password"] = json!("p w"); + spaced["allowUpdate"] = json!(true); + let (status, body) = call(&app, "POST", "/v1/peers/create", Some(spaced)).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["status"], "CREATED"); + assert_eq!( + at(&mock.cfg(), "source", "password").unwrap().as_str(), + Some("p w") + ); + + // malformed body → grpc-shaped 400 + let req = Request::builder() + .method("POST") + .uri("/v1/mirrors/status") + .header("content-type", "application/json") + .body(Full::new(Bytes::from("{not json"))) + .unwrap(); + let resp = handle(&app, req).await; + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let bytes = resp.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["code"], 3); + + // unknown mirror → 404 grpc shape + let (status, body) = call( + &app, + "POST", + "/v1/mirrors/status", + Some(json!({"flowJobName": "ghost"})), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + assert_eq!(body["code"], 5); + + // control daemon down → 503 + drop(mock); + let dir2 = tempfile::tempdir().unwrap(); + let downed = App { + control: ControlClient::new(dir2.path().join("missing.sock")), + store: Store::load(dir2.path().join("state.json")).await.unwrap(), + password: None, + version: "t".into(), + stats: Arc::new(walshadow_peerdb::stats::StatsHistory::new()), + }; + let (status, body) = call(&downed, "POST", "/v1/peers/create", Some(pg_peer("pg"))).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "{body}"); + assert_eq!(body["code"], 14); +} + +#[tokio::test] +async fn auth_gate() { + let dir = tempfile::tempdir().unwrap(); + let (app, _mock) = shim(dir.path(), Some("s3cret")).await; + + let (status, body) = call(&app, "GET", "/v1/version", None).await; + assert_eq!(status, StatusCode::UNAUTHORIZED); + assert_eq!(body["code"], 16); + + let req = Request::builder() + .method("GET") + .uri("/v1/version") + .header("authorization", "Bearer s3cret") + .body(Full::::default()) + .unwrap(); + let resp = handle(&app, req).await; + assert_eq!(resp.status(), StatusCode::OK); + + let req = Request::builder() + .method("GET") + .uri("/v1/version") + .header("authorization", "Bearer wrong") + .body(Full::::default()) + .unwrap(); + let resp = handle(&app, req).await; + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +}