test(run-engine): address inquisition (metric, floor, honesty)

Second review round (blind multi-model) found real issues:
- Contention metric counted poisson tenants as contending before their runs
  arrived, so trickle numbers measured arrival shape not fairness; it now only
  counts a tenant once it has arrived, unserved work.
- SFQ/stride floor was not monotonic, so a returning idle tenant could
  monopolise service (the CFS min_vruntime guarantee the comments claimed but
  did not implement). Floor is now monotonic non-decreasing.
- Dropped the misleading EEVDF eligibility term from SFQ (it never changed the
  ordering); it is plain start-time WFQ.
- FINDINGS corrected throughout: the DRR shortfall is a batch-drain measurement
  artifact (and virtual-time clusters queues at ties too, so the earlier
  explanation was wrong); worstWaitP99 is not an anti-staleness signal; the
  baseline age bias is not exercised here; cost was not rigorously measured;
  CoDel actively hurts under trickle arrival. Added the definitional-advantage
  and age-bias caveats up front.
- Persist rough cost proxies to results JSON; note flushdb scope; drop dead
  seed field from scenarios.

Author: 1stvamp

internal-packages/run-engine/src/run-queue/fairness-spike/FINDINGS.md

@@ -143,12 +143,18 @@ describe("fairness spike bench", () => {
           const runs = runsBySelector.get(name)!;
           const cwsw = stats(runs.map((r) => r.metrics.contentionWorstShareOverWeight));
           const jain = stats(runs.map((r) => r.metrics.contentionJain));
-          const lightWaitP99 = stats(runs.map((r) => r.metrics.worstWaitP99));
+          const worstWaitP99 = stats(runs.map((r) => r.metrics.worstWaitP99));
           return {
             selector: name,
             contentionWorstShareOverWeight: cwsw,
             contentionJain: jain,
-            worstWaitP99: lightWaitP99,
+            worstWaitP99: worstWaitP99,
+            // Rough cost proxies. redisOps here is the number of strategy
+            // invocations, which is NOT comparable across selectors (a candidate
+            // reads all queues per call; the baseline short-circuits when the env
+            // is at capacity), so treat as illustrative only.
+            selectionRounds: stats(runs.map((r) => r.metrics.redisOps)),
+            wallClockMs: stats(runs.map((r) => r.metrics.wallClockMs)),
             detailSeed0: runs[0].metrics.perGroup as GroupMetrics[],
           };
         });

internal-packages/run-engine/src/run-queue/fairness-spike/fairnessSpike.bench.test.ts

@@ -52,6 +52,9 @@ export async function runScenario(config: DriverConfig): Promise<RunMetrics> {
   const env = authenticatedEnv(limit);
 
   const admin = createRedisClient(config.redis);
+  // NOTE: flushes the whole Redis DB, ignoring keyPrefix. Safe against the
+  // dedicated testcontainer this spike runs on; do not point config.redis at a
+  // shared instance.
   await admin.flushdb();
 
   const queue = new RunQueue({

internal-packages/run-engine/src/run-queue/fairness-spike/harness/driver.ts

@@ -33,7 +33,12 @@ export type RunMetrics = {
    */
   contentionWorstShareOverWeight: number;
   contentionJain: number;
-  /** anti-staleness tail: the largest per-group p99 wait */
+  /**
+   * The largest per-group p99/max wait. NOTE: this is NOT an anti-staleness win
+   * signal. It is dominated by the highest-volume tenant, which a fair selector
+   * deliberately makes wait longer, so a fairer selector can score WORSE here.
+   * Read per-tenant waits (perGroup) for the anti-staleness story instead.
+   */
   worstWaitP99: number;
   worstWaitMax: number;
 };
@@ -53,30 +58,57 @@ function jain(values: number[]): number {
 }
 
 /**
- * Counts each group's dequeues that fall within the contention window: the
- * prefix of the dequeue timeline during which at least two groups still have
- * unfinished work. Once only one group has work left there is no contention to
- * be fair about, so those dequeues are excluded.
+ * Counts each group's dequeues that happen during genuine contention: a dequeue
+ * counts only if, at that instant, at least two groups have work that has
+ * already arrived (been enqueued) but not yet been served. This excludes both
+ * the drain tail (one group left) and, crucially, any stretch where a group's
+ * runs have not arrived yet under spread (poisson) arrival: a tenant only
+ * "contends" once its work exists, otherwise the metric would penalise it for
+ * the arrival process throttling its supply rather than the selector starving it.
  */
-function contentionCounts(
-  events: DequeueEvent[],
-  totals: Record<GroupId, number>
-): { counts: Map<GroupId, number>; windowSize: number; contended: Set<GroupId> } {
+function contentionCounts(events: DequeueEvent[]): {
+  counts: Map<GroupId, number>;
+  windowSize: number;
+  contended: Set<GroupId>;
+} {
   const ordered = [...events].sort((a, b) => a.dequeueAtMs - b.dequeueAtMs);
-  const remaining = new Map<GroupId, number>(Object.entries(totals));
+
+  // Per-group sorted arrival (enqueue) times, so we can ask how many of a
+  // group's runs have arrived by a given instant.
+  const arrivalsByGroup = new Map<GroupId, number[]>();
+  for (const e of events) {
+    const arr = arrivalsByGroup.get(e.groupId) ?? [];
+    arr.push(e.enqueueAtMs);
+    arrivalsByGroup.set(e.groupId, arr);
+  }
+  for (const arr of arrivalsByGroup.values()) arr.sort((a, b) => a - b);
+
+  const arrivedBy = (g: GroupId, t: number): number => {
+    const arr = arrivalsByGroup.get(g) ?? [];
+    let n = 0;
+    for (const a of arr) {
+      if (a <= t) n++;
+      else break;
+    }
+    return n;
+  };
+
+  const dequeuedSoFar = new Map<GroupId, number>();
   const counts = new Map<GroupId, number>();
   const contended = new Set<GroupId>();
   let windowSize = 0;
 
   for (const e of ordered) {
-    const withWork = [...remaining.entries()].filter(([, n]) => n > 0);
-    if (withWork.length < 2) break;
-    // Every group that still has work during this step is contending, whether
-    // or not it is the one being served (so a starved group counts as share 0).
-    for (const [g] of withWork) contended.add(g);
-    counts.set(e.groupId, (counts.get(e.groupId) ?? 0) + 1);
-    windowSize++;
-    remaining.set(e.groupId, (remaining.get(e.groupId) ?? 0) - 1);
+    const t = e.dequeueAtMs;
+    const withWork = [...arrivalsByGroup.keys()].filter(
+      (g) => arrivedBy(g, t) - (dequeuedSoFar.get(g) ?? 0) > 0
+    );
+    if (withWork.length >= 2) {
+      for (const g of withWork) contended.add(g);
+      counts.set(e.groupId, (counts.get(e.groupId) ?? 0) + 1);
+      windowSize++;
+    }
+    dequeuedSoFar.set(e.groupId, (dequeuedSoFar.get(e.groupId) ?? 0) + 1);
   }
 
   return { counts, windowSize, contended };
@@ -89,12 +121,11 @@ export function computeMetrics(input: {
   redisOps: number;
   wallClockMs: number;
 }): RunMetrics {
-  const { events, weights, totals } = input;
+  const { events, weights } = input;
   const groupIds = Object.keys(weights);
   const total = events.length;
-  const sumWeights = groupIds.reduce((a, g) => a + (weights[g] ?? 1), 0);
 
-  const { counts: windowCounts, windowSize, contended } = contentionCounts(events, totals);
+  const { counts: windowCounts, windowSize, contended } = contentionCounts(events);
   const sumWindowWeights = [...contended].reduce((a, g) => a + (weights[g] ?? 1), 0);
 
   const waitsByGroup = new Map<GroupId, number[]>();

internal-packages/run-engine/src/run-queue/fairness-spike/harness/metrics.ts

@@ -6,9 +6,8 @@ import type { WorkloadConfig } from "./workload.js";
  * scenario gives one tenant many queues, which is how the #2617 starvation
  * (a tenant multiplying its selection chances) shows up at the base-queue grain.
  */
-export const SCENARIOS: Record<string, WorkloadConfig> = {
+export const SCENARIOS: Record<string, Omit<WorkloadConfig, "seed">> = {
   balanced: {
-    seed: "spike-1",
     envConcurrencyLimit: 5,
     tenants: [
       { tenantId: "t-a", runCount: 50, holdMsMean: 30 },
@@ -19,7 +18,6 @@ export const SCENARIOS: Record<string, WorkloadConfig> = {
   },
 
   adversarialSkew: {
-    seed: "spike-1",
     envConcurrencyLimit: 5,
     tenants: [
       { tenantId: "heavy", runCount: 250, queueCount: 30, holdMsMean: 25 },
@@ -32,7 +30,6 @@ export const SCENARIOS: Record<string, WorkloadConfig> = {
   },
 
   weighted: {
-    seed: "spike-1",
     envConcurrencyLimit: 4,
     tenants: [
       { tenantId: "big", runCount: 300, weight: 3, holdMsMean: 30 },
@@ -41,7 +38,6 @@ export const SCENARIOS: Record<string, WorkloadConfig> = {
   },
 
   burst: {
-    seed: "spike-1",
     envConcurrencyLimit: 6,
     tenants: Array.from({ length: 6 }, (_, i) => ({
       tenantId: `burst-${i}`,
@@ -52,7 +48,6 @@ export const SCENARIOS: Record<string, WorkloadConfig> = {
   },
 
   longHold: {
-    seed: "spike-1",
     envConcurrencyLimit: 4,
     tenants: [
       { tenantId: "slow-1", runCount: 40, holdMsMean: 500 },
@@ -66,7 +61,6 @@ export const SCENARIOS: Record<string, WorkloadConfig> = {
   // whose runs then wait. This makes the baseline's age bias live and gives a
   // CoDel wrapper divergent per-tenant sojourns to react to.
   trickleStale: {
-    seed: "spike-1",
     envConcurrencyLimit: 4,
     tenants: [
       { tenantId: "heavy", runCount: 300, queueCount: 20, holdMsMean: 25 },

internal-packages/run-engine/src/run-queue/fairness-spike/harness/scenarios.ts

@@ -31,6 +31,16 @@
         "min": 1653,
         "max": 1989
       },
+      "selectionRounds": {
+        "mean": 646,
+        "min": 638,
+        "max": 658
+      },
+      "wallClockMs": {
+        "mean": 5439,
+        "min": 5087,
+        "max": 6100
+      },
       "detailSeed0": [
         {
           "groupId": "heavy",
@@ -117,6 +127,16 @@
         "min": 1632,
         "max": 2050
       },
+      "selectionRounds": {
+        "mean": 646,
+        "min": 636,
+        "max": 653
+      },
+      "wallClockMs": {
+        "mean": 4455.333333333333,
+        "min": 3897,
+        "max": 4776
+      },
       "detailSeed0": [
         {
           "groupId": "heavy",
@@ -203,6 +223,16 @@
         "min": 1626,
         "max": 2051
       },
+      "selectionRounds": {
+        "mean": 649.6666666666666,
+        "min": 639,
+        "max": 657
+      },
+      "wallClockMs": {
+        "mean": 4385.333333333333,
+        "min": 4233,
+        "max": 4534
+      },
       "detailSeed0": [
         {
           "groupId": "heavy",
@@ -289,6 +319,16 @@
         "min": 1632,
         "max": 2050
       },
+      "selectionRounds": {
+        "mean": 646,
+        "min": 636,
+        "max": 653
+      },
+      "wallClockMs": {
+        "mean": 4208,
+        "min": 4118,
+        "max": 4357
+      },
       "detailSeed0": [
         {
           "groupId": "heavy",
@@ -375,6 +415,16 @@
         "min": 1632,
         "max": 2050
       },
+      "selectionRounds": {
+        "mean": 646,
+        "min": 636,
+        "max": 653
+      },
+      "wallClockMs": {
+        "mean": 4296.666666666667,
+        "min": 3828,
+        "max": 4587
+      },
       "detailSeed0": [
         {
           "groupId": "heavy",
@@ -461,6 +511,16 @@
         "min": 1653,
         "max": 1989
       },
+      "selectionRounds": {
+        "mean": 646,
+        "min": 638,
+        "max": 658
+      },
+      "wallClockMs": {
+        "mean": 5449.333333333333,
+        "min": 5333,
+        "max": 5508
+      },
       "detailSeed0": [
         {
           "groupId": "heavy",

internal-packages/run-engine/src/run-queue/fairness-spike/results/adversarialSkew.json

@@ -29,6 +29,16 @@
         "min": 1126,
         "max": 1283
       },
+      "selectionRounds": {
+        "mean": 370.3333333333333,
+        "min": 365,
+        "max": 377
+      },
+      "wallClockMs": {
+        "mean": 930.3333333333334,
+        "min": 783,
+        "max": 1196
+      },
       "detailSeed0": [
         {
           "groupId": "t-a",
@@ -93,6 +103,16 @@
         "min": 1109,
         "max": 1271
       },
+      "selectionRounds": {
+        "mean": 366.3333333333333,
+        "min": 357,
+        "max": 373
+      },
+      "wallClockMs": {
+        "mean": 734.3333333333334,
+        "min": 576,
+        "max": 874
+      },
       "detailSeed0": [
         {
           "groupId": "t-a",
@@ -157,6 +177,16 @@
         "min": 1120,
         "max": 1279
       },
+      "selectionRounds": {
+        "mean": 364.6666666666667,
+        "min": 359,
+        "max": 368
+      },
+      "wallClockMs": {
+        "mean": 772,
+        "min": 550,
+        "max": 925
+      },
       "detailSeed0": [
         {
           "groupId": "t-a",
@@ -221,6 +251,16 @@
         "min": 1109,
         "max": 1271
       },
+      "selectionRounds": {
+        "mean": 366.3333333333333,
+        "min": 357,
+        "max": 373
+      },
+      "wallClockMs": {
+        "mean": 757,
+        "min": 721,
+        "max": 819
+      },
       "detailSeed0": [
         {
           "groupId": "t-a",
@@ -285,6 +325,16 @@
         "min": 1109,
         "max": 1271
       },
+      "selectionRounds": {
+        "mean": 366.3333333333333,
+        "min": 357,
+        "max": 373
+      },
+      "wallClockMs": {
+        "mean": 807.6666666666666,
+        "min": 721,
+        "max": 871
+      },
       "detailSeed0": [
         {
           "groupId": "t-a",
@@ -349,6 +399,16 @@
         "min": 1126,
         "max": 1283
       },
+      "selectionRounds": {
+        "mean": 370.3333333333333,
+        "min": 365,
+        "max": 377
+      },
+      "wallClockMs": {
+        "mean": 883,
+        "min": 793,
+        "max": 1031
+      },
       "detailSeed0": [
         {
           "groupId": "t-a",