|
| 1 | +import { redisTest } from "@internal/testcontainers"; |
| 2 | +import type { RedisOptions } from "@internal/redis"; |
| 3 | +import { describe } from "vitest"; |
| 4 | +import { mkdirSync, writeFileSync } from "node:fs"; |
| 5 | +import { dirname, join } from "node:path"; |
| 6 | +import { fileURLToPath } from "node:url"; |
| 7 | +import { runCkScenario } from "./harness/ckDriver.js"; |
| 8 | +import { |
| 9 | + buildWorkload, |
| 10 | + weightsOf, |
| 11 | + type WorkloadConfig, |
| 12 | +} from "../fairness-spike/harness/workload.js"; |
| 13 | +import type { RunMetrics, GroupMetrics } from "../fairness-spike/harness/metrics.js"; |
| 14 | +import { BaselineCk, SfqCk, DrrCk, type CkDiscipline } from "./disciplines.js"; |
| 15 | + |
| 16 | +/** |
| 17 | + * Caps vs scheduling. Runs the plan-of-record's concurrency CAPS (Phase-2 per-key |
| 18 | + * limit via the real per-queue concurrency gate, Phase-1 total cap via a |
| 19 | + * driver-side group gate) head-to-head against the scheduling disciplines |
| 20 | + * (SFQ/DRR) on identical scenarios, through the real CK-dequeue Lua at |
| 21 | + * maxCount = 1. |
| 22 | + * |
| 23 | + * A "treatment" is (order discipline, per-key cap?, total cap?). Caps are |
| 24 | + * admission settings, not disciplines: the per-key cap is the real Lua's native |
| 25 | + * per-variant gate (oldest-eligible-first, true age order), the total cap is the |
| 26 | + * driver refusing to admit past the group ceiling. |
| 27 | + * |
| 28 | + * Headline = the light (starved) key's wait. makespan = work-conservation signal. |
| 29 | + * contention share = directional. |
| 30 | + */ |
| 31 | + |
| 32 | +const RESULTS_DIR = join(dirname(fileURLToPath(import.meta.url)), "results"); |
| 33 | +const SEEDS = ["seed-a", "seed-b", "seed-c"]; |
| 34 | +const ENV_LIMIT = 4; |
| 35 | +const PER_KEY_CAP = 2; // heavy key(s) bound to half the env |
| 36 | +const TOTAL_CAP = 2; // per-task total ceiling, below env |
| 37 | + |
| 38 | +type Treatment = { |
| 39 | + label: string; |
| 40 | + makeDiscipline: () => CkDiscipline; |
| 41 | + perKeyCap?: number; |
| 42 | + totalCap?: number; |
| 43 | +}; |
| 44 | + |
| 45 | +const TREATMENTS: Treatment[] = [ |
| 46 | + { label: "baseline", makeDiscipline: () => new BaselineCk() }, |
| 47 | + { label: "perKeyCap", makeDiscipline: () => new BaselineCk(), perKeyCap: PER_KEY_CAP }, |
| 48 | + { label: "totalCap", makeDiscipline: () => new BaselineCk(), totalCap: TOTAL_CAP }, |
| 49 | + { label: "sfq", makeDiscipline: () => new SfqCk() }, |
| 50 | + { label: "drr", makeDiscipline: () => new DrrCk() }, |
| 51 | + { label: "perKeyCap+sfq", makeDiscipline: () => new SfqCk(), perKeyCap: PER_KEY_CAP }, |
| 52 | +]; |
| 53 | + |
| 54 | +type CapScenario = { |
| 55 | + config: Omit<WorkloadConfig, "seed">; |
| 56 | + /** the key whose wait is the headline (a starved light key, or the heavy key for heavy-idle) */ |
| 57 | + lightKey: string; |
| 58 | +}; |
| 59 | + |
| 60 | +function sybilHeavy(count: number, runsEach: number) { |
| 61 | + return Array.from({ length: count }, (_, i) => ({ |
| 62 | + tenantId: `heavy-${i}`, |
| 63 | + runCount: runsEach, |
| 64 | + holdMsMean: 25, |
| 65 | + })); |
| 66 | +} |
| 67 | + |
| 68 | +const SCENARIOS: Record<string, CapScenario> = { |
| 69 | + // one heavy key floods (old head), four light keys trickle in later. One heavy |
| 70 | + // key => a per-key cap frees slots the light keys can take. |
| 71 | + ckSkew: { |
| 72 | + lightKey: "light-1", |
| 73 | + config: { |
| 74 | + envConcurrencyLimit: ENV_LIMIT, |
| 75 | + tenants: [ |
| 76 | + { tenantId: "heavy", runCount: 240, holdMsMean: 25 }, |
| 77 | + { tenantId: "light-1", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, |
| 78 | + { tenantId: "light-2", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, |
| 79 | + { tenantId: "light-3", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, |
| 80 | + { tenantId: "light-4", runCount: 15, arrival: "poisson", ratePerSec: 10, holdMsMean: 25 }, |
| 81 | + ], |
| 82 | + }, |
| 83 | + }, |
| 84 | + |
| 85 | + // a bulk backlog plus two keys trickling in slowly |
| 86 | + ckTrickle: { |
| 87 | + lightKey: "trickle-1", |
| 88 | + config: { |
| 89 | + envConcurrencyLimit: ENV_LIMIT, |
| 90 | + tenants: [ |
| 91 | + { tenantId: "bulk", runCount: 240, holdMsMean: 25 }, |
| 92 | + { tenantId: "trickle-1", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, |
| 93 | + { tenantId: "trickle-2", runCount: 30, arrival: "poisson", ratePerSec: 25, holdMsMean: 25 }, |
| 94 | + ], |
| 95 | + }, |
| 96 | + }, |
| 97 | + |
| 98 | + // sybil split: one attacker spreads its backlog across 10 concurrency keys, each |
| 99 | + // with a large backlog that stays non-empty through the light key's whole |
| 100 | + // arrival window. Each attacker key is under the same per-key cap, but the cap |
| 101 | + // frees no aggregate slot (2 attacker keys fill env, and as one empties the next |
| 102 | + // attacker key's old head is served before the newer light key). Only a fair |
| 103 | + // order rescues the light key. |
| 104 | + ckSybil: { |
| 105 | + lightKey: "light", |
| 106 | + config: { |
| 107 | + envConcurrencyLimit: ENV_LIMIT, |
| 108 | + tenants: [ |
| 109 | + ...sybilHeavy(10, 30), |
| 110 | + { tenantId: "light", runCount: 20, arrival: "poisson", ratePerSec: 40, holdMsMean: 25 }, |
| 111 | + ], |
| 112 | + }, |
| 113 | + }, |
| 114 | + |
| 115 | + // work-conservation: one heavy key alone with a big backlog. A per-key or total |
| 116 | + // cap throttles it below the env limit and idles slots, inflating makespan; a |
| 117 | + // scheduler uses the whole env. Headline here is makespan, not wait. |
| 118 | + ckHeavyIdle: { |
| 119 | + lightKey: "heavy", |
| 120 | + config: { |
| 121 | + envConcurrencyLimit: ENV_LIMIT, |
| 122 | + tenants: [{ tenantId: "heavy", runCount: 200, holdMsMean: 25 }], |
| 123 | + }, |
| 124 | + }, |
| 125 | +}; |
| 126 | + |
| 127 | +function stats(xs: number[]) { |
| 128 | + return { |
| 129 | + mean: xs.reduce((a, b) => a + b, 0) / xs.length, |
| 130 | + min: Math.min(...xs), |
| 131 | + max: Math.max(...xs), |
| 132 | + }; |
| 133 | +} |
| 134 | + |
| 135 | +function fmt(n: number, d = 0): string { |
| 136 | + return Number.isFinite(n) ? n.toFixed(d) : String(n); |
| 137 | +} |
| 138 | + |
| 139 | +function waitOf(metrics: RunMetrics, key: string): number { |
| 140 | + return metrics.perGroup.find((g) => g.groupId === key)?.meanWait ?? 0; |
| 141 | +} |
| 142 | + |
| 143 | +function worstWaitOf(metrics: RunMetrics): number { |
| 144 | + return metrics.perGroup.length ? Math.max(...metrics.perGroup.map((g) => g.meanWait)) : 0; |
| 145 | +} |
| 146 | + |
| 147 | +describe("caps vs scheduling bench", () => { |
| 148 | + mkdirSync(RESULTS_DIR, { recursive: true }); |
| 149 | + |
| 150 | + for (const [scenarioName, scenario] of Object.entries(SCENARIOS)) { |
| 151 | + redisTest( |
| 152 | + `caps scenario: ${scenarioName}`, |
| 153 | + async ({ redisContainer }) => { |
| 154 | + const runs = new Map<string, Array<{ seed: string; metrics: RunMetrics }>>(); |
| 155 | + |
| 156 | + for (const seed of SEEDS) { |
| 157 | + const config: WorkloadConfig = { ...scenario.config, seed }; |
| 158 | + const workload = buildWorkload(config); |
| 159 | + const expectedTotal = workload.tenants.reduce((n, t) => n + t.runCount, 0); |
| 160 | + |
| 161 | + for (const treatment of TREATMENTS) { |
| 162 | + const redis: RedisOptions = { |
| 163 | + keyPrefix: `rq:caps:${scenarioName}:${treatment.label}:${seed}:`, |
| 164 | + host: redisContainer.getHost(), |
| 165 | + port: redisContainer.getPort(), |
| 166 | + }; |
| 167 | + const metrics = await runCkScenario({ |
| 168 | + redis, |
| 169 | + discipline: treatment.makeDiscipline(), |
| 170 | + workload, |
| 171 | + perKeyCap: treatment.perKeyCap, |
| 172 | + totalCap: treatment.totalCap, |
| 173 | + }); |
| 174 | + if (metrics.totalDequeued !== expectedTotal) { |
| 175 | + throw new Error( |
| 176 | + `${scenarioName}/${treatment.label}/${seed}: dequeued ${metrics.totalDequeued} of ${expectedTotal}` |
| 177 | + ); |
| 178 | + } |
| 179 | + const arr = runs.get(treatment.label) ?? []; |
| 180 | + arr.push({ seed, metrics }); |
| 181 | + runs.set(treatment.label, arr); |
| 182 | + } |
| 183 | + } |
| 184 | + |
| 185 | + const perTreatment = [...runs.entries()].map(([label, rs]) => ({ |
| 186 | + treatment: label, |
| 187 | + lightWait: stats(rs.map((r) => waitOf(r.metrics, scenario.lightKey))), |
| 188 | + worstWait: stats(rs.map((r) => worstWaitOf(r.metrics))), |
| 189 | + makespan: stats(rs.map((r) => r.metrics.makespanMs)), |
| 190 | + contentionWorst: stats(rs.map((r) => r.metrics.contentionWorstShareOverWeight)), |
| 191 | + detailSeed0: rs[0].metrics.perGroup as GroupMetrics[], |
| 192 | + })); |
| 193 | + |
| 194 | + const firstWorkload = buildWorkload({ ...scenario.config, seed: SEEDS[0] }); |
| 195 | + writeFileSync( |
| 196 | + join(RESULTS_DIR, `caps-${scenarioName}.json`), |
| 197 | + JSON.stringify( |
| 198 | + { |
| 199 | + scenario: scenarioName, |
| 200 | + seeds: SEEDS, |
| 201 | + envLimit: ENV_LIMIT, |
| 202 | + perKeyCap: PER_KEY_CAP, |
| 203 | + totalCap: TOTAL_CAP, |
| 204 | + lightKey: scenario.lightKey, |
| 205 | + weights: weightsOf(firstWorkload), |
| 206 | + perTreatment, |
| 207 | + }, |
| 208 | + null, |
| 209 | + 2 |
| 210 | + ) |
| 211 | + ); |
| 212 | + |
| 213 | + const lines = [ |
| 214 | + ``, |
| 215 | + `### caps: ${scenarioName} (${SEEDS.length} seeds, env=${ENV_LIMIT}, perKeyCap=${PER_KEY_CAP}, totalCap=${TOTAL_CAP}, light=${scenario.lightKey})`, |
| 216 | + `treatment lightWait worstWait makespan contWorstS/W`, |
| 217 | + ...perTreatment.map( |
| 218 | + (r) => |
| 219 | + `${r.treatment.padEnd(16)} ${fmt(r.lightWait.mean).padStart(8)} ${fmt( |
| 220 | + r.worstWait.mean |
| 221 | + ).padStart(8)} ${fmt(r.makespan.mean).padStart(7)} ${fmt( |
| 222 | + r.contentionWorst.mean, |
| 223 | + 3 |
| 224 | + ).padStart(7)}` |
| 225 | + ), |
| 226 | + ``, |
| 227 | + ]; |
| 228 | + process.stdout.write(lines.join("\n") + "\n"); |
| 229 | + }, |
| 230 | + 300_000 |
| 231 | + ); |
| 232 | + } |
| 233 | +}); |
0 commit comments