Ab#132322 improve back end performance

src/schema/query/recordsAggregation.query.ts

@@ -299,52 +299,61 @@ export default {
             ]);
           }
         }
-        // Fetch related fields from other forms
-        const relatedFields: any[] = await Form.aggregate([
-          {
-            $match: {
-              fields: {
-                $elemMatch: {
-                  resource: String(args.resource),
+        // Fetch related fields from other forms.
+        // Only needed when at least one `sourceField` is NOT found in the
+        // resource's own fields (i.e. is a candidate related-form field).
+        const hasUnknownSourceField = (sourceFields as string[]).some(
+          (fName) =>
+            !selectableDefaultRecordFieldsFlat.includes(fName) &&
+            !resource.fields.find((x) => x.name === fName)
+        );
+        const relatedFields: any[] = hasUnknownSourceField
+          ? await Form.aggregate([
+              {
+                $match: {
+                  fields: {
+                    $elemMatch: {
+                      resource: String(args.resource),
+                      $or: [
+                        {
+                          type: 'resource',
+                        },
+                        {
+                          type: 'resources',
+                        },
+                      ],
+                    },
+                  },
+                },
+              },
+              {
+                $unwind: '$fields',
+              },
+              {
+                $match: {
+                  'fields.resource': String(args.resource),
                   $or: [
                     {
-                      type: 'resource',
+                      'fields.type': 'resource',
                     },
                     {
-                      type: 'resources',
+                      'fields.type': 'resources',
                     },
                   ],
                 },
               },
-            },
-          },
-          {
-            $unwind: '$fields',
-          },
-          {
-            $match: {
-              'fields.resource': String(args.resource),
-              $or: [
-                {
-                  'fields.type': 'resource',
+              {
+                $addFields: {
+                  'fields.form': '$_id',
                 },
-                {
-                  'fields.type': 'resources',
+              },
+              {
+                $replaceRoot: {
+                  newRoot: '$fields',
                 },
-              ],
-            },
-          },
-          {
-            $addFields: {
-              'fields.form': '$_id',
-            },
-          },
-          {
-            $replaceRoot: {
-              newRoot: '$fields',
-            },
-          },
-        ]);
+              },
+            ])
+          : [];
         pipeline.push({
           $addFields: {
             record_id: {
@@ -358,6 +367,29 @@ export default {
           context.timeZone,
           context.user?.attributes || {}
         );
+        // Batch-load all referenceData documents referenced by the queried
+        // fields in a single query, instead of one `findById` per field.
+        const referenceDataIds = Array.from(
+          new Set(
+            sourceFields
+              .map((fName) => resource.fields.find((x) => x.name === fName))
+              .filter((f: any) => f && f.referenceData && f.referenceData.id)
+              .map((f: any) => f.referenceData.id)
+          )
+        );
+        const referenceDataById = new Map<string, any>();
+        if (referenceDataIds.length > 0) {
+          const refDataDocs = await ReferenceData.find({
+            _id: { $in: referenceDataIds },
+          }).populate({
+            path: 'apiConfiguration',
+            model: 'ApiConfiguration',
+            select: { name: 1, endpoint: 1, graphQLEndpoint: 1 },
+          });
+          for (const doc of refDataDocs) {
+            referenceDataById.set(String(doc._id), doc);
+          }
+        }
         // Loop on fields to apply lookups for special fields
         for (const fieldName of sourceFields) {
           const field = resource.fields.find((x) => x.name === fieldName);
@@ -494,20 +526,18 @@ export default {
           }
           // If we have referenceData fields
           if (field && field.referenceData && field.referenceData.id) {
-            const referenceData = await ReferenceData.findById(
-              field.referenceData.id
-            ).populate({
-              path: 'apiConfiguration',
-              model: 'ApiConfiguration',
-              select: { name: 1, endpoint: 1, graphQLEndpoint: 1 },
-            });
-            const referenceDataAggregation: any[] =
-              await buildReferenceDataAggregation(
-                referenceData,
-                field,
-                context
-              );
-            pipeline.push(...referenceDataAggregation);
+            const referenceData = referenceDataById.get(
+              String(field.referenceData.id)
+            );
+            if (referenceData) {
+              const referenceDataAggregation: any[] =
+                await buildReferenceDataAggregation(
+                  referenceData,
+                  field,
+                  context
+                );
+              pipeline.push(...referenceDataAggregation);
+            }
           }
         }
         pipeline.push({

src/server/apollo/dataSources.ts

@@ -9,20 +9,10 @@ import { ApolloServer } from '@apollo/server';
 import { Context } from './context';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import gql from 'graphql-tag';
-// eslint-disable-next-line import/no-extraneous-dependencies
-import { BaseRedisCache } from 'apollo-server-cache-redis';
-import Redis from 'ioredis';
-import config from 'config';
+import { getBaseCache } from '@utils/cache';
 
-/** Local storage initialization */
-const referenceDataCache = new BaseRedisCache({
-  client: new Redis(config.get('redis.url'), {
-    password: config.get('redis.password'),
-    showFriendlyErrorStack: true,
-    lazyConnect: true,
-    maxRetriesPerRequest: 5,
-  }),
-});
+/** Local storage initialization (shared base cache) */
+const referenceDataCache = getBaseCache();
 /** Local storage key for last request */
 const LAST_REQUEST_KEY = '_last_request';
 /** Property for filtering in requests */

src/utils/aggregation/setDisplayText.ts

@@ -16,38 +16,55 @@ const setDisplayText = async (
   resource: Resource,
   context: any
 ): Promise<void> => {
-  // Reducer to fetch fields with choices
-  const reducer = async (acc, x) => {
-    let lookAt = resource.fields;
-    let lookFor = x.value;
-    const [questionResource, question] = x.value.split('.');
+  // Cache resolved related Resource documents within this call to avoid
+  // repeated lookups when several mapped fields reference the same resource.
+  const resourceCache = new Map<string, Resource>();
+  const getRelatedResource = async (id: any): Promise<Resource> => {
+    const key = String(id);
+    if (resourceCache.has(key)) {
+      return resourceCache.get(key);
+    }
+    const doc = await Resource.findById(id);
+    resourceCache.set(key, doc);
+    return doc;
+  };
 
-    // in case it's a resource.s type question, search for the related resource
-    if (questionResource && question) {
-      const formResource = resource.fields.find(
-        (field: any) =>
-          questionResource === field.name &&
-          ['resource', 'resources'].includes(field.type)
-      );
-      if (formResource) {
-        lookAt = (await Resource.findById(formResource.resource)).fields;
-        lookFor = question;
+  // Resolve fields with choices in parallel
+  const resolved = await Promise.all(
+    mappedFields.map(async (x) => {
+      let lookAt = resource.fields;
+      let lookFor = x.value;
+      const [questionResource, question] = x.value.split('.');
+
+      // in case it's a resource.s type question, search for the related resource
+      if (questionResource && question) {
+        const formResource = resource.fields.find(
+          (field: any) =>
+            questionResource === field.name &&
+            ['resource', 'resources'].includes(field.type)
+        );
+        if (formResource) {
+          const related = await getRelatedResource(formResource.resource);
+          lookAt = related?.fields ?? [];
+          lookFor = question;
+        }
       }
+      // then, search for related field
+      const formField = lookAt.find((field: any) => {
+        return (
+          lookFor === field.name &&
+          (field.choices || field.choicesByUrl || field.choicesByGraphQL)
+        );
+      });
+      return formField ? { key: x.key, field: formField } : null;
+    })
+  );
+  const fieldWithChoices: Record<string, any> = {};
+  for (const entry of resolved) {
+    if (entry) {
+      fieldWithChoices[entry.key] = entry.field;
     }
-    // then, search for related field
-    const formField = lookAt.find((field: any) => {
-      return (
-        lookFor === field.name &&
-        (field.choices || field.choicesByUrl || field.choicesByGraphQL)
-      );
-    });
-    if (formField) {
-      return { ...(await acc), [x.key]: formField };
-    } else {
-      return { ...(await acc) };
-    }
-  };
-  const fieldWithChoices = await mappedFields.reduce(reducer, {});
+  }
   for (const [key, field] of Object.entries(fieldWithChoices)) {
     // Fetch choices from source ( static / rest / graphql )
     const choices = await getFullChoices(field, context);

src/utils/cache/index.ts

@@ -0,0 +1,97 @@
+import { BaseRedisCache } from 'apollo-server-cache-redis';
+import Redis from 'ioredis';
+import config from 'config';
+
+/**
+ * Shared Redis primitives reused across the app.
+ *
+ * Before this module, three places (`server/apollo/dataSources.ts`,
+ * `utils/form/getNextId.ts`, `utils/schema/resolvers/Query/all.ts`) each
+ * instantiated their own `ioredis` client + `BaseRedisCache`. This module
+ * centralizes that and exposes:
+ *
+ * - `getRedisClient()` — the shared `ioredis` instance (lazy connected).
+ * - `getBaseCache()`   — the shared `BaseRedisCache` (apollo-server-cache-redis).
+ * - `createCache(ns, ttl)` — a thin JSON-serializing wrapper with key
+ *   namespacing and a default TTL, for typed perf caches.
+ */
+
+/** Shared `ioredis` client. */
+let sharedClient: Redis | null = null;
+/**
+ * Returns the process-wide shared `ioredis` client (created lazily).
+ *
+ * @returns The shared `Redis` instance.
+ */
+export const getRedisClient = (): Redis => {
+  if (!sharedClient) {
+    sharedClient = new Redis(config.get<string>('redis.url'), {
+      password: config.get<string>('redis.password'),
+      showFriendlyErrorStack: true,
+      lazyConnect: true,
+      maxRetriesPerRequest: 5,
+    });
+  }
+  return sharedClient;
+};
+
+/** Shared `BaseRedisCache` (string KV with TTL) used by Apollo + utilities. */
+let sharedBase: BaseRedisCache | null = null;
+/**
+ * Returns the process-wide shared `BaseRedisCache` (created lazily) backed by
+ * `getRedisClient()`. Suitable for raw string-value usage (Apollo, getNextId,
+ * referenceData last-request).
+ *
+ * @returns The shared `BaseRedisCache` instance.
+ */
+export const getBaseCache = (): BaseRedisCache => {
+  if (!sharedBase) {
+    sharedBase = new BaseRedisCache({ client: getRedisClient() as any });
+  }
+  return sharedBase;
+};
+
+/**
+ * Typed JSON-serializing cache built on top of `getBaseCache()`.
+ *
+ * Only plain JSON-serializable values should be stored. Class instances
+ * (e.g. CASL `Ability`) must NOT be stored through this cache.
+ */
+export interface KVCache {
+  get<T = unknown>(key: string): Promise<T | undefined>;
+  set(key: string, value: unknown, ttlSeconds?: number): Promise<void>;
+  del(key: string): Promise<void>;
+}
+
+/**
+ * Create a namespaced cache with a default TTL.
+ *
+ * @param namespace Prefix prepended to every key (e.g. `permissionFilters`).
+ * @param defaultTTL Default TTL in seconds when `set` is called without one.
+ * @returns A `KVCache` that JSON-serializes values and namespaces keys.
+ */
+export const createCache = (namespace: string, defaultTTL: number): KVCache => {
+  const base = getBaseCache();
+  const ns = (k: string) => `${namespace}:${k}`;
+  return {
+    async get<T>(key: string) {
+      const raw = await base.get(ns(key));
+      if (raw === undefined || raw === null) {
+        return undefined;
+      }
+      try {
+        return JSON.parse(raw) as T;
+      } catch {
+        return undefined;
+      }
+    },
+    async set(key, value, ttlSeconds) {
+      await base.set(ns(key), JSON.stringify(value), {
+        ttl: ttlSeconds ?? defaultTTL,
+      });
+    },
+    async del(key) {
+      await base.delete(ns(key));
+    },
+  };
+};

src/utils/form/getNextId.ts

@@ -1,19 +1,10 @@
 import { Record, Form } from '@models';
 import mongoose from 'mongoose';
 import i18next from 'i18next';
-import { BaseRedisCache } from 'apollo-server-cache-redis';
-import Redis from 'ioredis';
-import config from 'config';
+import { getBaseCache } from '@utils/cache';
 
-/** Redis caching initialization */
-const nextIdCache = new BaseRedisCache({
-  client: new Redis(config.get('redis.url'), {
-    password: config.get('redis.password'),
-    showFriendlyErrorStack: true,
-    lazyConnect: true,
-    maxRetriesPerRequest: 5,
-  }),
-});
+/** Redis caching initialization (shared base cache) */
+const nextIdCache = getBaseCache();
 
 /** Default start padding size for the IDs */
 const PADDING_MAX_LENGTH = 8;