fix(replication): key logical replication leader lock on slot name (#4151)

## Problem

`LogicalReplicationClient` uses a Redlock leader lock to guarantee a
single active consumer per Postgres logical replication slot. The lock
resource was keyed on the client `name`:

```
logical-replication-client:${this.options.name}
```

A slot permits exactly one consumer, so the lock's job is to serialize
consumers **of a given slot**. Keying it on `name` breaks that whenever
two clients target the same slot with different names — most notably
across a rolling deploy where the client `name` changes but `slotName`
does not. Both acquire *distinct* locks, both consider themselves
leader, and the second to reach `START_REPLICATION` hits `replication
slot "<slot>" is active for PID <n>`. Because that query was
fire-and-forget and its failure was only logged (no retry), the consumer
stopped and replication stalled until the process was restarted.

## Fix

**1. Key the leader lock on `slotName`** — the actual single-consumer
resource:

```
logical-replication-client:${this.options.slotName}
```

Consumers of the same slot now contend on the same lock and hand off
cleanly across restarts/deploys; different slots stay independent.
`name` is kept for logging and the pg `application_name`.

**2. Self-healing resubscribe** (`resubscribeOnFailure`, opt-in) —
instead of logging-and-dying, a client re-subscribes with exponential
backoff after a lost election or a failed `START_REPLICATION`, so a
rolling deploy self-heals: the incoming pod retries until the draining
pod releases the slot, then takes over. Safety:
- `#cleanupAttempt()` unconditionally ends the pg client (freeing the
walsender) and releases the leader lock before rescheduling — retries
never leak connections/locks.
- `shutdown()` sets an intentional-stop latch re-checked after every
`await` in `subscribe()` (and aborts the lock-acquire spin), so a
resubscribe can never race or outlive an intentional shutdown.
- Backoff resets only on genuine stream start, so a permanently stuck
slot backs off to the ceiling and logs loudly rather than tight-looping;
an epoch guard neutralises stale `START_REPLICATION` catches.

Runs- and sessions-replication opt in and use `shutdown()` for all
intentional stops.

**3. Observability** — the admin runs-replication status route probed
the old name-keyed Redis key (would report `leader:false` for every
source after fix #1); now probes the slot-keyed key.

## Tests

`internal-packages/replication/src/client.test.ts` (real Postgres +
Redis containers):
- same-slot/different-name → second client must not double-lead or race
into "slot is active" (the regression)
- a failing `START_REPLICATION` retry loop must not leak connections or
locks
- `shutdown()` during an in-flight `subscribe()` must not leave a zombie
leader
- `subscribe()` after `shutdown()` re-arms `resubscribeOnFailure`
- self-heals once the leader releases the slot

Plus the multi-source wiring test updated to the slot-keyed lock keys.

## Rollout

With the self-healing resubscribe, this ships as a **plain rolling
deploy** — the incoming pods retry across the one-time lock-key
transition and take over once the old pods drain (a brief replication
stall that the durable slot replays on reconnect — no data loss). No
stop-before-start required.

Author: d-cs

.server-changes/fix-replication-leader-lock-per-slot.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+Key the logical-replication leader lock on the slot name (not the client name) so consumers of the same replication slot serialize correctly across restarts and rolling deploys

apps/webapp/app/routes/admin.api.v1.runs-replication.status.ts

@@ -7,12 +7,15 @@ import { getRunsReplicationConfiguredSources } from "~/services/runsReplicationG
 /**
  * Probes per-source replication leadership via the redlock leader-lock key, which
  * is DOUBLE-PREFIXED with `logical-replication-client:` — once from the connection's
- * keyPrefix and once from redlock's resource string. So we prefix this connection
- * with `runs-replication:logical-replication-client:` and EXISTS on the resource
- * `logical-replication-client:runs-replication:<id>`, resolving to:
- *   runs-replication:logical-replication-client:logical-replication-client:runs-replication:<id>
+ * keyPrefix and once from redlock's resource string. The lock is keyed on the
+ * replication slot, so we prefix this connection with
+ * `runs-replication:logical-replication-client:` and EXISTS on the resource
+ * `logical-replication-client:<slotName>`, resolving to:
+ *   runs-replication:logical-replication-client:logical-replication-client:<slotName>
  */
-async function probeLeadership(sourceIds: string[]): Promise<Map<string, boolean>> {
+async function probeLeadership(
+  sources: { id: string; slotName: string }[]
+): Promise<Map<string, boolean>> {
   const leaders = new Map<string, boolean>();
 
   const redis = new Redis({
@@ -26,9 +29,9 @@ async function probeLeadership(sourceIds: string[]): Promise<Map<string, boolean
   });
 
   try {
-    for (const id of sourceIds) {
-      const exists = await redis.exists(`logical-replication-client:runs-replication:${id}`);
-      leaders.set(id, exists === 1);
+    for (const source of sources) {
+      const exists = await redis.exists(`logical-replication-client:${source.slotName}`);
+      leaders.set(source.id, exists === 1);
     }
   } finally {
     await redis.quit();
@@ -46,7 +49,7 @@ export async function loader({ request }: LoaderFunctionArgs) {
     return json({ enabled: false, sources: [] });
   }
 
-  const leaders = await probeLeadership(sources.map((s) => s.id));
+  const leaders = await probeLeadership(sources);
 
   return json({
     enabled: env.RUN_REPLICATION_ENABLED === "1" && sources.length > 0,

apps/webapp/app/services/runsReplicationService.server.ts

@@ -286,6 +286,7 @@ export class RunsReplicationService {
         table: "TaskRun",
         redisOptions: options.redisOptions,
         autoAcknowledge: false,
+        resubscribeOnFailure: true,
         publicationActions: ["insert", "update", "delete"],
         logger:
           options.logger ?? new Logger("LogicalReplicationClient", options.logLevel ?? "info"),
@@ -428,7 +429,9 @@ export class RunsReplicationService {
 
     if (!hasCurrentTransaction) {
       this.logger.info("No transaction to commit, shutting down immediately");
-      await Promise.all(Array.from(this._sources.values()).map((runtime) => runtime.client.stop()));
+      await Promise.all(
+        Array.from(this._sources.values()).map((runtime) => runtime.client.shutdown())
+      );
       this._isShutDownComplete = true;
       return;
     }
@@ -458,7 +461,7 @@ export class RunsReplicationService {
     for (const runtime of this._sources.values()) {
       this.logger.info("Stopping replication client", { sourceId: runtime.source.id });
 
-      await runtime.client.stop();
+      await runtime.client.shutdown();
 
       if (runtime.acknowledgeInterval) {
         clearInterval(runtime.acknowledgeInterval);
@@ -636,7 +639,7 @@ export class RunsReplicationService {
       // swallow client.stop() rejections so they don't surface as unhandled.
       if (!this._shutdownStopInFlight) {
         this._shutdownStopInFlight = true;
-        Promise.all(Array.from(this._sources.values()).map((r) => r.client.stop()))
+        Promise.all(Array.from(this._sources.values()).map((r) => r.client.shutdown()))
           .catch((error) => {
             this.logger.error("Error stopping replication clients during shutdown", { error });
           })

apps/webapp/app/services/sessionsReplicationService.server.ts

@@ -187,6 +187,7 @@ export class SessionsReplicationService {
       table: "Session",
       redisOptions: options.redisOptions,
       autoAcknowledge: false,
+      resubscribeOnFailure: true,
       publicationActions: ["insert", "update", "delete"],
       logger: options.logger ?? new Logger("LogicalReplicationClient", options.logLevel ?? "info"),
       leaderLockTimeoutMs: options.leaderLockTimeoutMs ?? 30_000,
@@ -265,7 +266,7 @@ export class SessionsReplicationService {
 
     if (!this._currentTransaction) {
       this.logger.info("No transaction to commit, shutting down immediately");
-      await this._replicationClient.stop();
+      await this._replicationClient.shutdown();
       this._isSubscribed = false;
       this._isShutDownComplete = true;
       return;
@@ -294,7 +295,7 @@ export class SessionsReplicationService {
   async stop() {
     this.logger.info("Stopping replication client");
 
-    await this._replicationClient.stop();
+    await this._replicationClient.shutdown();
 
     if (this._acknowledgeInterval) {
       clearInterval(this._acknowledgeInterval);
@@ -430,10 +431,15 @@ export class SessionsReplicationService {
     if (this._isShutDownComplete) return;
 
     if (this._isShuttingDown) {
-      this._replicationClient.stop().finally(() => {
-        this._isSubscribed = false;
-        this._isShutDownComplete = true;
-      });
+      this._replicationClient
+        .shutdown()
+        .catch((error) => {
+          this.logger.error("Error stopping replication client during shutdown", { error });
+        })
+        .finally(() => {
+          this._isSubscribed = false;
+          this._isShutDownComplete = true;
+        });
     }
 
     // If there are no events, do nothing

apps/webapp/test/runsReplicationInstance.test.ts

@@ -408,10 +408,12 @@ describe("RunsReplication multi-source wiring (integration)", () => {
 
         probe = new Redis(redisOptions);
 
+        // Leader lock is keyed on the slot, so each source holds a distinct
+        // slot-keyed lock (double-prefixed: connection keyPrefix + redlock resource).
         const legacyKey =
-          "runs-replication:logical-replication-client:logical-replication-client:runs-replication:legacy";
+          "runs-replication:logical-replication-client:logical-replication-client:tr_legacy_wiring";
         const newKey =
-          "runs-replication:logical-replication-client:logical-replication-client:runs-replication:new";
+          "runs-replication:logical-replication-client:logical-replication-client:tr_new_wiring";
 
         expect(await probe.exists(legacyKey)).toBe(1);
         expect(await probe.exists(newKey)).toBe(1);