Skip to content

Commit 189a3b3

Browse files
myftijamyftija
committed
feat(webapp): separate rate limit budget for deployment endpoints
Most deploy-flow API calls (build-time env var resolution and sync, env key exchange, preview branches) drew from the same general API rate limit bucket as an environment's runtime traffic, so a busy environment could starve its own deployments; the /api/v*/deployments endpoints themselves were fully exempt from rate limits as a stopgap. The whole group now goes through a dedicated limiter with its own token bucket, keyed per environment for environment API keys and per token for PATs/OATs, configurable via DEPLOYMENT_RATE_LIMIT_* env vars. The general API limiter whitelists the group via a shared path list. /api/v1/deployments/current is runtime SDK surface and stays exempt from rate limits as before, outside the deploy budget.
1 parent 820c079 commit 189a3b3

9 files changed

Lines changed: 164 additions & 1 deletion

File tree

‎.server-changes/deployment-rate-limit-budget.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
area: webapp
3+
type: improvement
4+
---
5+
6+
Deployment-related API endpoints now draw from their own generous rate limit budget, configurable via the `DEPLOYMENT_RATE_LIMIT_*` environment variables, so runtime API traffic no longer competes with deployments for the same per-environment budget.

‎apps/webapp/app/entry.server.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ singleton("SentryTenantContextProcessor", () => {
302302
});
303303

304304
export { apiRateLimiter } from "./services/apiRateLimit.server";
305+
export { deploymentRateLimiter } from "./services/deploymentRateLimit.server";
305306
export { engineRateLimiter } from "./services/engineRateLimit.server";
306307
export { otlpRateLimiter } from "./services/otlpRateLimit.server";
307308
export { runWithHttpContext } from "./services/httpAsyncStorage.server";

‎apps/webapp/app/env.server.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -614,6 +614,14 @@ const EnvironmentSchema = z
614614
API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"),
615615
API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60),
616616

617+
// Separate budget for deploy-flow endpoints, see deploymentRateLimit.server.ts
618+
DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"),
619+
DEPLOYMENT_RATE_LIMIT_MAX: z.coerce.number().int().default(1500),
620+
DEPLOYMENT_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(500),
621+
DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
622+
DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
623+
DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED: z.string().default("0"),
624+
617625
// Per-IP rate limit for the unauthenticated OTLP ingestion endpoints
618626
// (/otel/*). Bounds unauthenticated request rates. Opt-in
619627
// (disabled by default): because it keys on the source IP, it is only

‎apps/webapp/app/services/apiRateLimit.server.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { resolvePrivateApiKeyRateLimitScope } from "~/models/runtimeEnvironment.
44
import { batchStreamGrants } from "~/runEngine/concerns/batchStreamGrantsInstance.server";
55
import { authenticateAuthorizationHeader } from "./apiAuth.server";
66
import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server";
7+
import { deploymentApiPaths } from "./deploymentApiPaths.server";
78
import type { Duration } from "./rateLimiter.server";
89

910
const BATCH_STREAM_ITEMS_PATH = /^\/api\/v3\/batches\/([^/]+)\/items$/;
@@ -91,7 +92,8 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
9192
"/api/v1/auth/jwt/claims",
9293
/^\/api\/v1\/runs\/[^/]+\/attempts$/, // /api/v1/runs/$runFriendlyId/attempts
9394
/^\/api\/v1\/waitpoints\/tokens\/[^/]+\/callback\/[^/]+$/, // /api/v1/waitpoints/tokens/$waitpointFriendlyId/callback/$hash
94-
/^\/api\/v\d+\/deployments/, // /api/v{1,2,3,n}/deployments/*
95+
...deploymentApiPaths, // rate limited separately by deploymentRateLimiter
96+
/^\/api\/v\d+\/deployments\/current$/, // runtime SDK surface, exempt as before the deploy budget split
9597
// Internal SDK plumbing — packets are presigned-URL handshakes for
9698
// payload uploads (v2 PUT) and downloads (v1 GET), authenticated via
9799
// run-scoped JWT, called once per task/turn boundary by the runtime.

‎apps/webapp/app/services/deploymentApiPaths.server.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
// Deploy-flow endpoints, rate limited by deploymentRateLimiter with a separate
2+
// budget instead of the general per-environment buckets.
3+
export const deploymentApiPaths: (RegExp | string)[] = [
4+
// /current is runtime SDK surface, kept out of the deploy budget
5+
/^\/api\/v\d+\/deployments(?!\/current$)(\/|$)/,
6+
/^\/api\/v1\/projects\/[^/]+\/(dev|staging|prod|preview)$/,
7+
/^\/api\/v1\/projects\/[^/]+\/envvars$/,
8+
/^\/api\/v1\/projects\/[^/]+\/envvars\/[^/]+\/import$/,
9+
/^\/api\/v1\/projects\/[^/]+\/branches$/,
10+
/^\/api\/v1\/projects\/[^/]+\/branches\/archive$/,
11+
"/api/v1/remote-build-provider-status",
12+
"/api/v1/artifacts",
13+
];

‎apps/webapp/app/services/deploymentRateLimit.server.ts‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import { env } from "~/env.server";
2+
import { resolvePrivateApiKeyRateLimitScope } from "~/models/runtimeEnvironment.server";
3+
import { authorizationRateLimitMiddleware } from "./authorizationRateLimitMiddleware.server";
4+
import { deploymentApiPaths } from "./deploymentApiPaths.server";
5+
import type { Duration } from "./rateLimiter.server";
6+
7+
export const deploymentRateLimiter = authorizationRateLimitMiddleware({
8+
redis: {
9+
port: env.RATE_LIMIT_REDIS_PORT,
10+
host: env.RATE_LIMIT_REDIS_HOST,
11+
username: env.RATE_LIMIT_REDIS_USERNAME,
12+
password: env.RATE_LIMIT_REDIS_PASSWORD,
13+
tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true",
14+
clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1",
15+
},
16+
keyPrefix: "deployment",
17+
defaultLimiter: {
18+
type: "tokenBucket",
19+
refillRate: env.DEPLOYMENT_RATE_LIMIT_REFILL_RATE,
20+
interval: env.DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL as Duration,
21+
maxTokens: env.DEPLOYMENT_RATE_LIMIT_MAX,
22+
},
23+
limiterCache: {
24+
fresh: 60_000 * 10,
25+
stale: 60_000 * 20,
26+
maxItems: 1000,
27+
},
28+
limiterConfigOverride: async (authorizationValue) => {
29+
const rawApiKey = authorizationValue.replace(/^Bearer /, "");
30+
31+
if (!rawApiKey.startsWith("tr_")) {
32+
return;
33+
}
34+
35+
const scope = await resolvePrivateApiKeyRateLimitScope(rawApiKey);
36+
37+
if (!scope) {
38+
return;
39+
}
40+
41+
// Identifier only: the org's apiRateLimiterConfig governs the general API
42+
// limiter, not the deploy budget.
43+
return {
44+
identifier: scope.environmentId,
45+
};
46+
},
47+
pathMatchers: deploymentApiPaths,
48+
log: {
49+
rejections: env.DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",
50+
requests: env.DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED === "1",
51+
limiter: env.DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED === "1",
52+
},
53+
});

‎apps/webapp/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@ async function startServer() {
183183
const socketIo: { io: IoServer } | undefined = build.entry.module.socketIo;
184184
const wss: WebSocketServer | undefined = build.entry.module.wss;
185185
const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter;
186+
const deploymentRateLimiter: RateLimitMiddleware = build.entry.module.deploymentRateLimiter;
186187
const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter;
187188
const otlpRateLimiter: RequestHandler = build.entry.module.otlpRateLimiter;
188189
const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext;
@@ -235,6 +236,7 @@ async function startServer() {
235236
}
236237

237238
app.use(apiRateLimiter);
239+
app.use(deploymentRateLimiter);
238240
app.use(engineRateLimiter);
239241
app.use(otlpRateLimiter);
240242

‎apps/webapp/test/deploymentApiPaths.test.ts‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import { describe, expect, it } from "vitest";
2+
import { deploymentApiPaths } from "../app/services/deploymentApiPaths.server.js";
3+
4+
// Same matching semantics as authorizationRateLimitMiddleware's pathMatchers/pathWhiteList
5+
function matchesAnyPath(path: string, matchers: (RegExp | string)[]): boolean {
6+
return matchers.some((matcher) =>
7+
matcher instanceof RegExp ? matcher.test(path) : path === matcher
8+
);
9+
}
10+
11+
describe("deploymentApiPaths", () => {
12+
it("matches every endpoint the deploy flow calls", () => {
13+
const deployFlowPaths = [
14+
"/api/v1/deployments",
15+
"/api/v1/deployments/latest",
16+
"/api/v1/deployments/deployment_123",
17+
"/api/v1/deployments/deployment_123/progress",
18+
"/api/v1/deployments/deployment_123/fail",
19+
"/api/v1/deployments/deployment_123/cancel",
20+
"/api/v1/deployments/deployment_123/background-workers",
21+
"/api/v1/deployments/deployment_123/generate-registry-credentials",
22+
"/api/v1/deployments/20260811.1/promote",
23+
"/api/v3/deployments/deployment_123/finalize",
24+
"/api/v1/projects/proj_abc123/dev",
25+
"/api/v1/projects/proj_abc123/staging",
26+
"/api/v1/projects/proj_abc123/prod",
27+
"/api/v1/projects/proj_abc123/preview",
28+
"/api/v1/projects/proj_abc123/envvars",
29+
"/api/v1/projects/proj_abc123/envvars/prod/import",
30+
"/api/v1/projects/proj_abc123/branches",
31+
"/api/v1/projects/proj_abc123/branches/archive",
32+
"/api/v1/remote-build-provider-status",
33+
"/api/v1/artifacts",
34+
];
35+
36+
for (const path of deployFlowPaths) {
37+
expect(
38+
matchesAnyPath(path, deploymentApiPaths),
39+
`expected ${path} to be a deployment API path`
40+
).toBe(true);
41+
}
42+
});
43+
44+
it("does not match runtime API surface", () => {
45+
const runtimePaths = [
46+
"/api/v1/deployments/current",
47+
"/api/v1/deploymentsfoo",
48+
"/api/v1/whoami",
49+
"/api/v2/whoami",
50+
"/api/v1/tasks/my-task/trigger",
51+
"/api/v1/tasks/batch",
52+
"/api/v2/runs/run_123",
53+
"/api/v1/runs/run_123/replay",
54+
"/api/v3/runs/run_123/trace",
55+
"/api/v1/projects",
56+
"/api/v1/projects/proj_abc123",
57+
"/api/v1/projects/proj_abc123/dev-status",
58+
"/api/v1/projects/proj_abc123/prod/jwt",
59+
"/api/v1/projects/proj_abc123/envvars/prod",
60+
"/api/v1/projects/proj_abc123/envvars/prod/MY_VAR",
61+
"/api/v1/schedules",
62+
"/api/v1/queues/queue_123",
63+
];
64+
65+
for (const path of runtimePaths) {
66+
expect(
67+
matchesAnyPath(path, deploymentApiPaths),
68+
`expected ${path} not to be a deployment API path`
69+
).toBe(false);
70+
}
71+
});
72+
});

‎docs/self-hosting/env/webapp.mdx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ mode: "wide"
7676
| `API_RATE_LIMIT_LIMITER_LOGS_ENABLED` | No | 0 | API rate limit limiter logs. |
7777
| `API_RATE_LIMIT_JWT_WINDOW` | No | 1m | API rate limit JWT window. |
7878
| `API_RATE_LIMIT_JWT_TOKENS` | No | 60 | API rate limit JWT tokens. |
79+
| `DEPLOYMENT_RATE_LIMIT_REFILL_INTERVAL` | No | 10s | Deployment endpoints rate limit refill interval. |
80+
| `DEPLOYMENT_RATE_LIMIT_MAX` | No | 1500 | Deployment endpoints rate limit max. |
81+
| `DEPLOYMENT_RATE_LIMIT_REFILL_RATE` | No | 500 | Deployment endpoints rate limit refill rate. |
82+
| `DEPLOYMENT_RATE_LIMIT_REQUEST_LOGS_ENABLED` | No | 0 | Deployment endpoints rate limit request logs. |
83+
| `DEPLOYMENT_RATE_LIMIT_REJECTION_LOGS_ENABLED` | No | 1 | Deployment endpoints rate limit rejection logs. |
84+
| `DEPLOYMENT_RATE_LIMIT_LIMITER_LOGS_ENABLED` | No | 0 | Deployment endpoints rate limit limiter logs. |
7985
| **Deploy & Registry** | | | |
8086
| `DEPLOY_REGISTRY_HOST` | Yes | — | Deploy registry host. |
8187
| `DEPLOY_REGISTRY_USERNAME` | No | — | Deploy registry username. |

0 commit comments

Comments
 (0)