diff --git a/src/schema/query/recordsAggregation.query.ts b/src/schema/query/recordsAggregation.query.ts index 47a1c33e1..dc5fb215d 100644 --- a/src/schema/query/recordsAggregation.query.ts +++ b/src/schema/query/recordsAggregation.query.ts @@ -299,52 +299,61 @@ export default { ]); } } - // Fetch related fields from other forms - const relatedFields: any[] = await Form.aggregate([ - { - $match: { - fields: { - $elemMatch: { - resource: String(args.resource), + // Fetch related fields from other forms. + // Only needed when at least one `sourceField` is NOT found in the + // resource's own fields (i.e. is a candidate related-form field). + const hasUnknownSourceField = (sourceFields as string[]).some( + (fName) => + !selectableDefaultRecordFieldsFlat.includes(fName) && + !resource.fields.find((x) => x.name === fName) + ); + const relatedFields: any[] = hasUnknownSourceField + ? await Form.aggregate([ + { + $match: { + fields: { + $elemMatch: { + resource: String(args.resource), + $or: [ + { + type: 'resource', + }, + { + type: 'resources', + }, + ], + }, + }, + }, + }, + { + $unwind: '$fields', + }, + { + $match: { + 'fields.resource': String(args.resource), $or: [ { - type: 'resource', + 'fields.type': 'resource', }, { - type: 'resources', + 'fields.type': 'resources', }, ], }, }, - }, - }, - { - $unwind: '$fields', - }, - { - $match: { - 'fields.resource': String(args.resource), - $or: [ - { - 'fields.type': 'resource', + { + $addFields: { + 'fields.form': '$_id', }, - { - 'fields.type': 'resources', + }, + { + $replaceRoot: { + newRoot: '$fields', }, - ], - }, - }, - { - $addFields: { - 'fields.form': '$_id', - }, - }, - { - $replaceRoot: { - newRoot: '$fields', - }, - }, - ]); + }, + ]) + : []; pipeline.push({ $addFields: { record_id: { @@ -358,6 +367,29 @@ export default { context.timeZone, context.user?.attributes || {} ); + // Batch-load all referenceData documents referenced by the queried + // fields in a single query, instead of one `findById` per field. + const referenceDataIds = Array.from( + new Set( + sourceFields + .map((fName) => resource.fields.find((x) => x.name === fName)) + .filter((f: any) => f && f.referenceData && f.referenceData.id) + .map((f: any) => f.referenceData.id) + ) + ); + const referenceDataById = new Map(); + if (referenceDataIds.length > 0) { + const refDataDocs = await ReferenceData.find({ + _id: { $in: referenceDataIds }, + }).populate({ + path: 'apiConfiguration', + model: 'ApiConfiguration', + select: { name: 1, endpoint: 1, graphQLEndpoint: 1 }, + }); + for (const doc of refDataDocs) { + referenceDataById.set(String(doc._id), doc); + } + } // Loop on fields to apply lookups for special fields for (const fieldName of sourceFields) { const field = resource.fields.find((x) => x.name === fieldName); @@ -494,20 +526,18 @@ export default { } // If we have referenceData fields if (field && field.referenceData && field.referenceData.id) { - const referenceData = await ReferenceData.findById( - field.referenceData.id - ).populate({ - path: 'apiConfiguration', - model: 'ApiConfiguration', - select: { name: 1, endpoint: 1, graphQLEndpoint: 1 }, - }); - const referenceDataAggregation: any[] = - await buildReferenceDataAggregation( - referenceData, - field, - context - ); - pipeline.push(...referenceDataAggregation); + const referenceData = referenceDataById.get( + String(field.referenceData.id) + ); + if (referenceData) { + const referenceDataAggregation: any[] = + await buildReferenceDataAggregation( + referenceData, + field, + context + ); + pipeline.push(...referenceDataAggregation); + } } } pipeline.push({ diff --git a/src/server/apollo/dataSources.ts b/src/server/apollo/dataSources.ts index fb5691558..cd5b90d23 100644 --- a/src/server/apollo/dataSources.ts +++ b/src/server/apollo/dataSources.ts @@ -9,20 +9,10 @@ import { ApolloServer } from '@apollo/server'; import { Context } from './context'; // eslint-disable-next-line import/no-extraneous-dependencies import gql from 'graphql-tag'; -// eslint-disable-next-line import/no-extraneous-dependencies -import { BaseRedisCache } from 'apollo-server-cache-redis'; -import Redis from 'ioredis'; -import config from 'config'; +import { getBaseCache } from '@utils/cache'; -/** Local storage initialization */ -const referenceDataCache = new BaseRedisCache({ - client: new Redis(config.get('redis.url'), { - password: config.get('redis.password'), - showFriendlyErrorStack: true, - lazyConnect: true, - maxRetriesPerRequest: 5, - }), -}); +/** Local storage initialization (shared base cache) */ +const referenceDataCache = getBaseCache(); /** Local storage key for last request */ const LAST_REQUEST_KEY = '_last_request'; /** Property for filtering in requests */ diff --git a/src/utils/aggregation/setDisplayText.ts b/src/utils/aggregation/setDisplayText.ts index eb11c2cd8..fe8183b95 100644 --- a/src/utils/aggregation/setDisplayText.ts +++ b/src/utils/aggregation/setDisplayText.ts @@ -16,38 +16,55 @@ const setDisplayText = async ( resource: Resource, context: any ): Promise => { - // Reducer to fetch fields with choices - const reducer = async (acc, x) => { - let lookAt = resource.fields; - let lookFor = x.value; - const [questionResource, question] = x.value.split('.'); + // Cache resolved related Resource documents within this call to avoid + // repeated lookups when several mapped fields reference the same resource. + const resourceCache = new Map(); + const getRelatedResource = async (id: any): Promise => { + const key = String(id); + if (resourceCache.has(key)) { + return resourceCache.get(key); + } + const doc = await Resource.findById(id); + resourceCache.set(key, doc); + return doc; + }; - // in case it's a resource.s type question, search for the related resource - if (questionResource && question) { - const formResource = resource.fields.find( - (field: any) => - questionResource === field.name && - ['resource', 'resources'].includes(field.type) - ); - if (formResource) { - lookAt = (await Resource.findById(formResource.resource)).fields; - lookFor = question; + // Resolve fields with choices in parallel + const resolved = await Promise.all( + mappedFields.map(async (x) => { + let lookAt = resource.fields; + let lookFor = x.value; + const [questionResource, question] = x.value.split('.'); + + // in case it's a resource.s type question, search for the related resource + if (questionResource && question) { + const formResource = resource.fields.find( + (field: any) => + questionResource === field.name && + ['resource', 'resources'].includes(field.type) + ); + if (formResource) { + const related = await getRelatedResource(formResource.resource); + lookAt = related?.fields ?? []; + lookFor = question; + } } + // then, search for related field + const formField = lookAt.find((field: any) => { + return ( + lookFor === field.name && + (field.choices || field.choicesByUrl || field.choicesByGraphQL) + ); + }); + return formField ? { key: x.key, field: formField } : null; + }) + ); + const fieldWithChoices: Record = {}; + for (const entry of resolved) { + if (entry) { + fieldWithChoices[entry.key] = entry.field; } - // then, search for related field - const formField = lookAt.find((field: any) => { - return ( - lookFor === field.name && - (field.choices || field.choicesByUrl || field.choicesByGraphQL) - ); - }); - if (formField) { - return { ...(await acc), [x.key]: formField }; - } else { - return { ...(await acc) }; - } - }; - const fieldWithChoices = await mappedFields.reduce(reducer, {}); + } for (const [key, field] of Object.entries(fieldWithChoices)) { // Fetch choices from source ( static / rest / graphql ) const choices = await getFullChoices(field, context); diff --git a/src/utils/cache/index.ts b/src/utils/cache/index.ts new file mode 100644 index 000000000..c8f16e3f5 --- /dev/null +++ b/src/utils/cache/index.ts @@ -0,0 +1,97 @@ +import { BaseRedisCache } from 'apollo-server-cache-redis'; +import Redis from 'ioredis'; +import config from 'config'; + +/** + * Shared Redis primitives reused across the app. + * + * Before this module, three places (`server/apollo/dataSources.ts`, + * `utils/form/getNextId.ts`, `utils/schema/resolvers/Query/all.ts`) each + * instantiated their own `ioredis` client + `BaseRedisCache`. This module + * centralizes that and exposes: + * + * - `getRedisClient()` — the shared `ioredis` instance (lazy connected). + * - `getBaseCache()` — the shared `BaseRedisCache` (apollo-server-cache-redis). + * - `createCache(ns, ttl)` — a thin JSON-serializing wrapper with key + * namespacing and a default TTL, for typed perf caches. + */ + +/** Shared `ioredis` client. */ +let sharedClient: Redis | null = null; +/** + * Returns the process-wide shared `ioredis` client (created lazily). + * + * @returns The shared `Redis` instance. + */ +export const getRedisClient = (): Redis => { + if (!sharedClient) { + sharedClient = new Redis(config.get('redis.url'), { + password: config.get('redis.password'), + showFriendlyErrorStack: true, + lazyConnect: true, + maxRetriesPerRequest: 5, + }); + } + return sharedClient; +}; + +/** Shared `BaseRedisCache` (string KV with TTL) used by Apollo + utilities. */ +let sharedBase: BaseRedisCache | null = null; +/** + * Returns the process-wide shared `BaseRedisCache` (created lazily) backed by + * `getRedisClient()`. Suitable for raw string-value usage (Apollo, getNextId, + * referenceData last-request). + * + * @returns The shared `BaseRedisCache` instance. + */ +export const getBaseCache = (): BaseRedisCache => { + if (!sharedBase) { + sharedBase = new BaseRedisCache({ client: getRedisClient() as any }); + } + return sharedBase; +}; + +/** + * Typed JSON-serializing cache built on top of `getBaseCache()`. + * + * Only plain JSON-serializable values should be stored. Class instances + * (e.g. CASL `Ability`) must NOT be stored through this cache. + */ +export interface KVCache { + get(key: string): Promise; + set(key: string, value: unknown, ttlSeconds?: number): Promise; + del(key: string): Promise; +} + +/** + * Create a namespaced cache with a default TTL. + * + * @param namespace Prefix prepended to every key (e.g. `permissionFilters`). + * @param defaultTTL Default TTL in seconds when `set` is called without one. + * @returns A `KVCache` that JSON-serializes values and namespaces keys. + */ +export const createCache = (namespace: string, defaultTTL: number): KVCache => { + const base = getBaseCache(); + const ns = (k: string) => `${namespace}:${k}`; + return { + async get(key: string) { + const raw = await base.get(ns(key)); + if (raw === undefined || raw === null) { + return undefined; + } + try { + return JSON.parse(raw) as T; + } catch { + return undefined; + } + }, + async set(key, value, ttlSeconds) { + await base.set(ns(key), JSON.stringify(value), { + ttl: ttlSeconds ?? defaultTTL, + }); + }, + async del(key) { + await base.delete(ns(key)); + }, + }; +}; diff --git a/src/utils/form/getNextId.ts b/src/utils/form/getNextId.ts index 19d5a6edf..682dfd6b9 100644 --- a/src/utils/form/getNextId.ts +++ b/src/utils/form/getNextId.ts @@ -1,19 +1,10 @@ import { Record, Form } from '@models'; import mongoose from 'mongoose'; import i18next from 'i18next'; -import { BaseRedisCache } from 'apollo-server-cache-redis'; -import Redis from 'ioredis'; -import config from 'config'; +import { getBaseCache } from '@utils/cache'; -/** Redis caching initialization */ -const nextIdCache = new BaseRedisCache({ - client: new Redis(config.get('redis.url'), { - password: config.get('redis.password'), - showFriendlyErrorStack: true, - lazyConnect: true, - maxRetriesPerRequest: 5, - }), -}); +/** Redis caching initialization (shared base cache) */ +const nextIdCache = getBaseCache(); /** Default start padding size for the IDs */ const PADDING_MAX_LENGTH = 8; diff --git a/src/utils/schema/resolvers/Query/all.ts b/src/utils/schema/resolvers/Query/all.ts index fac40bd49..bb7a22c28 100644 --- a/src/utils/schema/resolvers/Query/all.ts +++ b/src/utils/schema/resolvers/Query/all.ts @@ -1,7 +1,7 @@ import { GraphQLError, valueFromASTUntyped } from 'graphql'; import { Record, ReferenceData, User } from '@models'; import extendAbilityForRecords from '@security/extendAbilityForRecords'; -import { decodeCursor, encodeCursor } from '@schema/types'; +import { encodeCursor } from '@schema/types'; import getReversedFields from '../../introspection/getReversedFields'; import getFilter, { FLAT_DEFAULT_FIELDS, @@ -21,6 +21,7 @@ import { accessibleBy } from '@casl/mongoose'; import { graphQLAuthCheck } from '@schema/shared'; import NodeCache from 'node-cache'; import { AppAbility } from '@security/defineUserAbility'; +import { createCache } from '@utils/cache'; /** Default number for items to get */ const DEFAULT_FIRST = 25; @@ -28,6 +29,13 @@ const DEFAULT_FIRST = 25; /** Ability Cache, based on user id, time to live: 5min */ const abilityCache = new NodeCache({ stdTTL: 60 * 5, checkperiod: 60 }); +/** + * Permission filters cache, keyed by user id. 5 min TTL. + * Backed by Redis when configured (shared across instances), otherwise + * by an in-process map. Values are plain mongo filter objects. + */ +const permissionFiltersCache = createCache('permissionFilters', 60 * 5); + // todo: improve by only keeping used fields in the $project stage /** * Project aggregation. @@ -223,7 +231,6 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => sortOrder = 'asc', first = DEFAULT_FIRST, skip = 0, - afterCursor, filter = {}, display = false, styles = [], @@ -424,18 +431,16 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => if (!ability) { // If not available, build ability ability = await extendAbilityForRecords(user); - set(context, 'user.ability', ability); - permissionFilters = Record.find( - accessibleBy(ability, 'read').Record - ).getFilter(); - // And cache it abilityCache.set(userId, ability); - } else { - // Update user ability - set(context, 'user.ability', ability); + } + set(context, 'user.ability', ability); + // Try to get permission filters from cache (Redis or memory) + permissionFilters = await permissionFiltersCache.get(userId); + if (!permissionFilters) { permissionFilters = Record.find( accessibleBy(ability, 'read').Record ).getFilter(); + await permissionFiltersCache.set(userId, permissionFilters); } // Finally putting all filters together @@ -482,43 +487,10 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => }); items = aggregation[0].items; totalCount = aggregation[0]?.totalCount[0]?.count || 0; - } else { - // If we're using cursors, get pagination filters <---- DEPRECATED ?? - const cursorFilters = afterCursor - ? { - _id: { - $gt: decodeCursor(afterCursor), - }, - } - : {}; - const pipeline = [ - { $match: basicFilters }, - ...linkedRecordsAggregation, - ...linkedReferenceDataAggregation, - { $match: { $and: [filters, cursorFilters] } }, - { - $facet: { - results: [ - ...(await getSortAggregation( - sortField, - sortOrder, - fields, - context - )), - { $limit: first + 1 }, - ], - totalCount: [ - { - $count: 'count', - }, - ], - }, - }, - ]; - const aggregation = await Record.aggregate(pipeline); - items = aggregation[0].items; - totalCount = aggregation[0]?.totalCount[0]?.count || 0; } + // Note: the cursor-based pagination branch was removed: it was deprecated, + // dead (read `aggregation[0].items` from a facet aliased `results`) and + // unreachable since callers always provide a `skip` value (defaults to 0). // When a sub-selection passes a `filter` argument, we cannot satisfy it // from the bulk pre-fetch below (it groups records by id only). Let the @@ -596,7 +568,6 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => record?: any; records?: any[]; }[] = []; - const relatedFilters = []; for (const item of items as any) { item._relatedRecords = {}; item.data = item.data || {}; @@ -619,15 +590,41 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => } for (const field of relatedFields) { itemsToUpdate.push({ item, field }); - relatedFilters.push({ - $or: [ - { resource: idsByName[field.entityName] }, - { form: idsByName[field.entityName] }, - ], - [`data.${field.name}`]: item.id, - }); } } + // Group related filters by (entityName, fieldName) so each related + // field results in a single `$or` clause with `$in: [...itemIds]` + // instead of one `$or` clause per item. + const relatedFiltersByKey = new Map< + string, + { entityName: string; fieldName: string; itemIds: any[] } + >(); + if (relatedFields.length > 0) { + for (const item of items as any) { + for (const field of relatedFields) { + const key = `${field.entityName}::${field.name}`; + let entry = relatedFiltersByKey.get(key); + if (!entry) { + entry = { + entityName: field.entityName, + fieldName: field.name, + itemIds: [], + }; + relatedFiltersByKey.set(key, entry); + } + entry.itemIds.push(item.id); + } + } + } + const relatedFilters = Array.from(relatedFiltersByKey.values()).map( + (entry) => ({ + $or: [ + { resource: idsByName[entry.entityName] }, + { form: idsByName[entry.entityName] }, + ], + [`data.${entry.fieldName}`]: { $in: entry.itemIds }, + }) + ); // Extract unique IDs const relatedIds = [ ...new Set( @@ -652,7 +649,7 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => archived: { $ne: true }, }, projection - ); + ).lean(); // Update items for (const item of itemsToUpdate) { if (item.record) { @@ -701,26 +698,28 @@ export default (entityName: string, fieldsByName: any, idsByName: any) => if (styles?.length > 0) { // Create the filter for each style const recordsIds = items.map((x) => x.id || x._id); - for (const style of styles) { - const styleFilter = getFilter(style.filter, fields, context); - // Get the records corresponding to the style filter - const itemsToStyle = await Record.aggregate([ - { - $match: { - _id: { - $in: recordsIds.map((x) => new mongoose.Types.ObjectId(x)), + const objectIds = recordsIds.map((x) => new mongoose.Types.ObjectId(x)); + // Run style filters in parallel + const styleResults = await Promise.all( + styles.map((style) => { + const styleFilter = getFilter(style.filter, fields, context); + return Record.aggregate([ + { + $match: { + _id: { $in: objectIds }, }, }, - }, - ...calculatedFieldsAggregation, - { - $match: styleFilter, - }, - { $addFields: { id: '$_id' } }, - ]); - // Add the list of record and the corresponding style - styleRules.push({ items: itemsToStyle, style: style }); - } + ...calculatedFieldsAggregation, + { + $match: styleFilter, + }, + { $addFields: { id: '$_id' } }, + ]); + }) + ); + styles.forEach((style, idx) => { + styleRules.push({ items: styleResults[idx], style }); + }); } // === ACTIONS ===