fix: correctly categorize errors related to saturated project databases

src/http/error-handler.ts

@@ -1,7 +1,7 @@
 import { FastifyError } from '@fastify/error'
 import { ErrorCode, isRenderableError, StorageBackendError, StorageError } from '@internal/errors'
+import { isDatabaseSlowDownError } from '@internal/errors/database-error'
 import { FastifyInstance } from 'fastify'
-import { DatabaseError } from 'pg'
 
 /**
  * The global error handler for all the uncaught exceptions within a request.
@@ -24,17 +24,7 @@ export const setErrorHandler = (
     request.executionError = error
 
     // database error
-    if (
-      error instanceof DatabaseError &&
-      [
-        'Authentication error', // supavisor specific
-        'Max client connections reached',
-        'remaining connection slots are reserved for non-replication superuser connections',
-        'no more connections allowed',
-        'sorry, too many clients already',
-        'server login has been failing, try again later',
-      ].some((msg) => (error as DatabaseError).message.includes(msg))
-    ) {
+    if (isDatabaseSlowDownError(error)) {
       return reply.status(429).send(
         formatter({
           statusCode: `429`,

src/http/routes/s3/error-handler.ts

@@ -1,9 +1,9 @@
 import { S3ServiceException } from '@aws-sdk/client-s3'
 import { FastifyError } from '@fastify/error'
 import { ErrorCode, StorageBackendError } from '@internal/errors'
+import { isDatabaseSlowDownError } from '@internal/errors/database-error'
 import { FastifyReply } from 'fastify/types/reply'
 import { FastifyRequest } from 'fastify/types/request'
-import { DatabaseError } from 'pg'
 
 type ValidationIssue = {
   instancePath?: string
@@ -46,16 +46,7 @@ export const s3ErrorHandler = (
   }
 
   // database error
-  if (
-    error instanceof DatabaseError &&
-    [
-      'Max client connections reached',
-      'remaining connection slots are reserved for non-replication superuser connections',
-      'no more connections allowed',
-      'sorry, too many clients already',
-      'server login has been failing, try again later',
-    ].some((msg) => (error as DatabaseError).message.includes(msg))
-  ) {
+  if (isDatabaseSlowDownError(error)) {
     return reply.status(429).send({
       Error: {
         Resource: resource,

src/internal/database/connection.ts

@@ -135,15 +135,6 @@ export class TenantConnection {
         throw ERRORS.DatabaseTimeout(e)
       }
 
-      // Handle database connection limit errors
-      if (
-        e instanceof DatabaseError &&
-        ((e.code === '08P01' && e.message.includes('no more connections allowed')) ||
-          e.message.includes('Max client connections reached'))
-      ) {
-        throw ERRORS.DatabaseConnectionLimit(e)
-      }
-
       throw e
     }
   }

src/internal/errors/codes.ts

@@ -20,7 +20,6 @@ export enum ErrorCode {
   KeyAlreadyExists = 'KeyAlreadyExists',
   BucketAlreadyExists = 'BucketAlreadyExists',
   DatabaseTimeout = 'DatabaseTimeout',
-  DatabaseConnectionLimit = 'DatabaseConnectionLimit',
   DatabaseReadOnly = 'DatabaseReadOnly',
   DatabaseInvalidObjectDefinition = 'DatabaseInvalidObjectDefinition',
   DatabaseSchemaMismatch = 'DatabaseSchemaMismatch',
@@ -386,15 +385,6 @@ export const ERRORS = {
       originalError: e,
     }),
 
-  DatabaseConnectionLimit: (e?: Error) =>
-    new StorageBackendError({
-      code: ErrorCode.DatabaseConnectionLimit,
-      httpStatusCode: 503,
-      message:
-        'The database has reached its maximum number of connections. Please try again later.',
-      originalError: e,
-    }),
-
   DatabaseReadOnly: (e?: Error) =>
     new StorageBackendError({
       code: ErrorCode.DatabaseReadOnly,

src/internal/errors/database-error.ts

@@ -0,0 +1,17 @@
+import { DatabaseError } from 'pg'
+
+export function isDatabaseSlowDownError(error: Error): boolean {
+  return (
+    error instanceof DatabaseError &&
+    [
+      'Authentication error', // supavisor specific
+      'Max client connections reached',
+      'remaining connection slots are reserved for non-replication superuser connections',
+      'no more connections allowed',
+      'sorry, too many clients already',
+      'server login has been failing, try again later',
+      'server login has been failing, cached error: connect timeout (server_login_retry)',
+      'server login has been failing, cached error: the database system is not accepting connections (server_login_retry)',
+    ].some((msg) => error.message.includes(msg))
+  )
+}