feat(assets): add extractAssets function for asset mapping across Con…

upload-api/migration-contentful/index.js

@@ -5,11 +5,13 @@ const createInitialMapper = require('./libs/createInitialMapper');
 const extractLocale = require('./libs/extractLocale');
 const extractTaxonomy = require('./libs/extractTaxonomy');
 const extractEntries = require('./libs/extractEntries');
+const extractAssets = require('./libs/extractAssets');
 
 module.exports = {
   extractContentTypes,
   createInitialMapper,
   extractLocale,
   extractTaxonomy,
-  extractEntries
+  extractEntries,
+  extractAssets
 };

upload-api/migration-contentful/libs/extractAssets.js

@@ -0,0 +1,106 @@
+'use strict';
+/* eslint-disable @typescript-eslint/no-var-requires */
+
+const { readFile } = require('../utils/helper');
+
+/**
+ * Extracts asset mapping rows from a Contentful export.
+ *
+ * Discovery mirrors extractEntries in this same package: the Contentful export
+ * is a single JSON file read via readFile(cleanLocalPath), with top-level
+ * `entries`, `locales` and `assets` arrays (standard `contentful-export`
+ * output). Each asset carries a stable `sys.id` and locale-keyed fields, so
+ * file metadata lives under `fields.file[locale]` and the title under
+ * `fields.title[locale]`:
+ *
+ *   {
+ *     sys: { id: '<stable-asset-id>' },
+ *     fields: {
+ *       title: { 'en-US': 'My Asset' },
+ *       file: {
+ *         'en-US': {
+ *           fileName: 'my-asset.png',
+ *           url: '//images.ctfassets.net/.../my-asset.png',
+ *           details: { size: 12345 }
+ *         }
+ *       }
+ *     }
+ *   }
+ *
+ * We dedupe by sys.id and use it as both `id` and `otherCmsAssetUid` so the row
+ * stays stable across delta iterations. Returned rows are consumed by
+ * putTestData on the main API to populate the asset_mapper DB (AssetMapper UI),
+ * matching the AEM/Sitecore AssetMappingRow shape.
+ *
+ * @param {string} cleanLocalPath - Path to the Contentful export JSON file.
+ * @returns {Array<{id:string,otherCmsAssetUid:string,filename:string,title:string,file_size:(number|string),assetPath:string,isUpdate:boolean}>}
+ */
+const extractAssets = (cleanLocalPath) => {
+  try {
+    const alldata = readFile(cleanLocalPath);
+    const assets = alldata?.assets;
+    const locales = Array.isArray(alldata?.locales)
+      ? alldata?.locales?.map((locale) => locale?.code).filter(Boolean)
+      : [];
+
+    if (!assets || !Array?.isArray(assets) || assets?.length === 0) {
+      console.info('No assets found in Contentful export');
+      return [];
+    }
+
+    const rows = [];
+    const seenIds = new Set();
+
+    // Prefer a locale-keyed value, falling back to the first available locale.
+    const pickLocalized = (field) => {
+      if (!field || typeof field !== 'object') return undefined;
+      for (const locale of locales) {
+        if (field[locale] !== undefined) return field[locale];
+      }
+      const firstKey = Object.keys(field)[0];
+      return firstKey !== undefined ? field[firstKey] : undefined;
+    };
+
+    for (const asset of assets) {
+      const id = asset?.sys?.id;
+      if (!id || seenIds.has(id)) {
+        continue;
+      }
+
+      const file = pickLocalized(asset?.fields?.file);
+      const titleValue = pickLocalized(asset?.fields?.title);
+      const title = typeof titleValue === 'string' ? titleValue : '';
+
+      const filename =
+        (typeof file?.fileName === 'string' && file?.fileName) || title || '';
+      const fileSize = file?.details?.size ?? '';
+      // Contentful serves assets from a protocol-relative CDN URL ("//...");
+      // normalize to https so downstream consumers get an absolute URL.
+      let assetPath = typeof file?.url === 'string' ? file?.url : '';
+      if (assetPath.startsWith('//')) {
+        assetPath = `https:${assetPath}`;
+      }
+
+      seenIds.add(id);
+
+      rows.push({
+        id,
+        otherCmsAssetUid: id,
+        filename,
+        title: title || filename,
+        file_size: fileSize,
+        assetPath,
+        isUpdate: false,
+      });
+    }
+
+    console.info(`extractAssets: Extracted ${rows.length} Contentful assets`);
+
+    return rows;
+  } catch (err) {
+    console.error('Error extracting Contentful assets:', err);
+    return [];
+  }
+};
+
+module.exports = extractAssets;

upload-api/migration-drupal/index.js

@@ -4,11 +4,13 @@ const extractTaxonomy = require('./libs/extractTaxonomy');
 const createInitialMapper = require('./libs/createInitialMapper');
 const extractLocale = require('./libs/extractLocale');
 const extractEntries = require('./libs/extractEntries');
+const extractAssets = require('./libs/extractAssets');
 
 module.exports = {
   // extractContentTypes,
   extractTaxonomy,
   createInitialMapper,
   extractLocale,
-  extractEntries
+  extractEntries,
+  extractAssets
 };

upload-api/migration-drupal/libs/extractAssets.js

@@ -0,0 +1,109 @@
+'use strict';
+/* eslint-disable @typescript-eslint/no-var-requires */
+
+/**
+ * Builds the assetMapping array consumed by putTestData on the main API to
+ * populate the asset_mapper DB shown on the AssetMapper UI — same flow/shape
+ * as AEM/Sitecore/Contentful/WordPress.
+ *
+ * Drupal reads from MySQL (not files). Files live in the `file_managed` table
+ * (fid, uuid, filename, uri, filesize, filemime). We surface every managed
+ * file, deduped by its Drupal file id, using the fid as the stable
+ * id/otherCmsAssetUid so it lines up across delta iterations.
+ *
+ * Row shape mirrors migration-sitecore/libs/extractAssets.js:
+ *   { id, otherCmsAssetUid, filename, title, file_size, assetPath, isUpdate }
+ */
+
+const { dbConnection } = require('../utils/helper');
+
+/**
+ * Derive a display-friendly path from the Drupal file uri.
+ * e.g. "public://2023-01/photo.jpg" -> "public/2023-01"
+ */
+const assetPathFromUri = (uri, filename) => {
+  if (!uri || typeof uri !== 'string') {
+    return '';
+  }
+  // Strip the stream wrapper scheme ("public://", "private://", "s3://", ...)
+  let cleaned = uri.replace(/^[a-z0-9]+:\/\//i, '');
+  // Drop the filename to leave only the directory portion
+  if (filename && cleaned.endsWith(filename)) {
+    cleaned = cleaned.slice(0, cleaned.length - filename.length);
+  }
+  return cleaned.replace(/\/+$/, '');
+};
+
+/**
+ * @param {Object} config - Migration config; must contain config.mysql (DB
+ *   connection details). config.assetsConfig is accepted but unused for now.
+ * @returns {Promise<Array<object>>} assetMapping rows
+ */
+const extractAssets = async (config) => {
+  const rows = [];
+  let connection;
+
+  try {
+    connection = dbConnection(config);
+
+    // Bound the result set so a very large media library can't be pulled fully into
+    // memory. Configurable via assetsConfig.maxAssets; parameterized to keep it out of
+    // the SQL string.
+    const maxAssets = Number(config?.assetsConfig?.maxAssets) || 100000;
+    const query = `
+      SELECT fid, uuid, filename, uri, filesize, filemime
+      FROM file_managed
+      ORDER BY fid
+      LIMIT ?
+    `;
+    const [results] = await connection.promise().query(query, [maxAssets]);
+
+    if (Array.isArray(results) && results.length === maxAssets) {
+      console.warn(
+        `extractAssets (Drupal): hit maxAssets cap of ${maxAssets}; some assets may be omitted`
+      );
+    }
+
+    const seenIds = new Set();
+
+    for (const file of results || []) {
+      if (!file) {
+        continue;
+      }
+
+      const id = file?.fid != null ? String(file?.fid) : '';
+      if (!id || seenIds.has(id)) {
+        continue;
+      }
+      seenIds.add(id);
+
+      const filename = file?.filename ? String(file?.filename) : `File ${id}`;
+
+      rows.push({
+        id,
+        otherCmsAssetUid: id,
+        filename,
+        title: filename,
+        file_size: file?.filesize != null ? String(file?.filesize) : '',
+        assetPath: assetPathFromUri(file?.uri, filename),
+        isUpdate: false
+      });
+    }
+
+    console.info(`extractAssets (Drupal): ${rows.length} asset(s)`);
+    return rows;
+  } catch (err) {
+    console.error('extractAssets (Drupal) error:', err?.message || err);
+    return rows;
+  } finally {
+    if (connection) {
+      try {
+        connection.end();
+      } catch (endErr) {
+        console.error('extractAssets (Drupal) connection close error:', endErr?.message || endErr);
+      }
+    }
+  }
+};
+
+module.exports = extractAssets;

upload-api/migration-wordpress/index.ts

@@ -2,10 +2,12 @@ import extractContentTypes from './libs/contentTypes';
 //import contentTypeMaker from './libs/contentTypeMapper';
 import extractLocale from './libs/extractLocale';
 import extractEntries from './libs/extractEntries';
+import extractAssets from './libs/extractAssets';
 
 export {
     extractContentTypes,
     //contentTypeMaker,
     extractLocale,
-    extractEntries
+    extractEntries,
+    extractAssets
 }

upload-api/migration-wordpress/libs/extractAssets.ts

@@ -0,0 +1,97 @@
+import fs from 'fs';
+
+export interface AssetMappingRow {
+  id: string;
+  otherCmsAssetUid: string;
+  filename: string;
+  title: string;
+  file_size: number | string;
+  assetPath: string;
+  isUpdate: boolean;
+}
+
+const normalizeArray = <T>(value: T | T[] | undefined): T[] => {
+  if (!value) return [];
+  return Array.isArray(value) ? value : [value];
+};
+
+/**
+ * WordPress WXR exports are parsed upstream into JSON before reaching here, so
+ * we load the file exactly like extractEntries/extractItems do: read the file
+ * and JSON.parse it, then walk rss.channel.item. Media are `item` elements with
+ * `wp:post_type === 'attachment'`.
+ */
+const getAssetId = (item: any): string => {
+  const postId = item?.['wp:post_id'];
+  if (postId != null && String(postId).trim() !== '') {
+    return String(postId).trim();
+  }
+  const guid = item?.guid?.text ?? item?.guid;
+  return guid != null ? String(guid).trim() : '';
+};
+
+const getAssetUrl = (item: any): string => {
+  const url = item?.['wp:attachment_url'] ?? item?.guid?.text ?? item?.guid;
+  return url != null ? String(url).trim() : '';
+};
+
+const getFilename = (item: any, assetUrl: string): string => {
+  const fromUrl = assetUrl?.split?.('/')?.pop?.();
+  if (typeof fromUrl === 'string' && fromUrl.trim() !== '') {
+    return fromUrl.trim();
+  }
+  const title = item?.title?.text ?? item?.title;
+  return typeof title === 'string' ? title.trim() : '';
+};
+
+const getTitle = (item: any, filename: string): string => {
+  const title = item?.title?.text ?? item?.title;
+  if (typeof title === 'string' && title.trim() !== '') {
+    return title.trim();
+  }
+  return filename.split('.').slice(0, -1).join('.') || filename;
+};
+
+const extractAssets = async (filePath: string): Promise<AssetMappingRow[]> => {
+  const rows: AssetMappingRow[] = [];
+  try {
+    const rawData = await fs.promises.readFile(filePath, 'utf8');
+    const jsonData = JSON.parse(rawData);
+    const items = normalizeArray(jsonData?.rss?.channel?.item);
+
+    const seenIds = new Set<string>();
+
+    for (const item of items) {
+      if (item?.['wp:post_type'] !== 'attachment') {
+        continue;
+      }
+
+      const id = getAssetId(item);
+      if (!id || seenIds.has(id)) {
+        continue;
+      }
+      seenIds.add(id);
+
+      const assetPath = getAssetUrl(item);
+      const filename = getFilename(item, assetPath);
+      const title = getTitle(item, filename);
+
+      rows.push({
+        id,
+        otherCmsAssetUid: id,
+        filename,
+        title,
+        file_size: '',
+        assetPath,
+        isUpdate: false,
+      });
+    }
+
+    return rows;
+  } catch (error: any) {
+    console.error('Error while extracting WordPress assets:', error?.message || error);
+    return rows;
+  }
+};
+
+export default extractAssets;

upload-api/src/controllers/wordpress/index.ts

@@ -2,7 +2,7 @@ import axios from "axios";
 import logger from "../../utils/logger";
 import { HTTP_CODES, HTTP_TEXTS, MIGRATION_DATA_CONFIG } from "../../constants";
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-import { extractContentTypes, extractLocale, extractEntries } from 'migration-wordpress';
+import { extractContentTypes, extractLocale, extractEntries, extractAssets } from 'migration-wordpress';
 import { deleteFolderSync } from "../../helper";
 import path from "path";
 
@@ -44,7 +44,8 @@ const createWordpressMapper = async (filePath: string = "", projectId: string |
     }
 
     if(contentTypeData){
-      const fieldMapping: any = { contentTypes: [], extractPath: filePath };
+      const assetMapping = await extractAssets(filePath);
+      const fieldMapping: any = { contentTypes: [], extractPath: filePath, assetMapping };
       contentTypeData.forEach((contentType: any) => {
         const jsonfileContent = contentType;
         jsonfileContent.type = "content_type";