perf(export): stream large exports in bounded chunks instead of buffe…

src/lib/export-scheduler/__tests__/exporter.test.ts

@@ -6,6 +6,15 @@ import { describe, it, expect } from 'vitest';
 import { exportData, fetchDataForTemplate } from '../exporter';
 import { ExportTemplate } from '../types';
 
+function readBlobText(blob: Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(String(reader.result));
+    reader.onerror = () => reject(reader.error);
+    reader.readAsText(blob);
+  });
+}
+
 describe('Data Exporter', () => {
   const mockTemplate: ExportTemplate = {
     id: 'test-1',
@@ -31,14 +40,23 @@ describe('Data Exporter', () => {
       expect(result.blob).toBeInstanceOf(Blob);
       expect(result.fileName).toContain('.csv');
       expect(result.blob.type).toContain('text/csv');
+
+      const text = await readBlobText(result.blob);
+      expect(text.split('\n')[0]).toBe('id,name,value');
+      expect(text).toContain('Item 2');
+      expect(text).toContain('Item 1');
     });
 
-    it('should export to JSON', async () => {
+    it('should export to JSON as an equivalent, valid array', async () => {
       const template = { ...mockTemplate, format: 'json' as const };
       const result = await exportData(template, mockData);
       expect(result.blob).toBeInstanceOf(Blob);
       expect(result.fileName).toContain('.json');
       expect(result.blob.type).toContain('application/json');
+
+      const text = await readBlobText(result.blob);
+      expect(JSON.parse(text)).toEqual(mockData.rows);
+      expect(text).toBe(JSON.stringify(mockData.rows, null, 2));
     });
 
     it('should export to XLSX', async () => {

src/lib/export-scheduler/exporter.ts

@@ -5,7 +5,15 @@
 
 import { createLogger } from '@/lib/logging';
 import { createCounterMetric, measureAsync } from '@/lib/logging/performance';
-import { ExportExecutionOptions, emitProgress, prepareExportData } from '@/lib/export';
+import {
+  createCSVSnapshot,
+  createJSONSnapshot,
+  escapeHtml,
+  escapeXml,
+  ExportExecutionOptions,
+  emitProgress,
+  prepareExportData,
+} from '@/lib/export';
 import { ExportFormat, ExportTemplate } from './types';
 
 export interface ExportData {
@@ -87,72 +95,66 @@ export async function exportData(
 }
 
 async function exportToCSV(data: ExportData): Promise<Blob> {
-  const { headers, rows } = data;
-
-  const escape = (value: unknown): string => {
-    const str = String(value ?? '');
-    if (str.includes(',') || str.includes('"') || str.includes('\n')) {
-      return `"${str.replace(/"/g, '""')}"`;
-    }
-    return str;
-  };
-
-  const csvLines: string[] = [];
-  csvLines.push(headers.map(escape).join(','));
-
-  for (const row of rows) {
-    const values = headers.map((header) => escape(row[header]));
-    csvLines.push(values.join(','));
+  const parts: string[] = [];
+  for await (const chunk of createCSVSnapshot(data)) {
+    parts.push(chunk.join('\n'));
   }
 
-  return new Blob([csvLines.join('\n')], { type: 'text/csv;charset=utf-8;' });
+  return new Blob([parts.join('\n')], { type: 'text/csv;charset=utf-8;' });
 }
 
 async function exportToJSON(data: ExportData): Promise<Blob> {
-  return new Blob([JSON.stringify(data.rows, null, 2)], {
-    type: 'application/json;charset=utf-8;',
-  });
+  const parts: string[] = [];
+  for await (const chunk of createJSONSnapshot(data)) {
+    parts.push(chunk);
+  }
+
+  return new Blob(parts, { type: 'application/json;charset=utf-8;' });
 }
 
 async function exportToXLSX(data: ExportData): Promise<Blob> {
   const { headers, rows } = data;
 
-  let xml = '<?xml version="1.0"?>\n';
-  xml += '<Workbook xmlns="urn:schemas-microsoft-com:office:spreadsheet" ';
-  xml += 'xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet">\n';
-  xml += '  <Worksheet ss:Name="Sheet1">\n';
-  xml += '    <Table>\n';
-  xml += '      <Row>\n';
-
-  for (const header of headers) {
-    xml += `        <Cell><Data ss:Type="String">${escapeXml(header)}</Data></Cell>\n`;
-  }
-
-  xml += '      </Row>\n';
-
+  const headerRow = headers
+    .map((header) => `<Cell><Data ss:Type="String">${escapeXml(header)}</Data></Cell>`)
+    .join('\n        ');
+  const rowChunks: string[] = [];
   for (const row of rows) {
-    xml += '      <Row>\n';
-    for (const header of headers) {
-      const value = row[header];
-      const type = typeof value === 'number' ? 'Number' : 'String';
-      xml += `        <Cell><Data ss:Type="${type}">${escapeXml(
-        String(value ?? ''),
-      )}</Data></Cell>\n`;
-    }
-    xml += '      </Row>\n';
+    rowChunks.push(
+      `      <Row>\n${headers
+        .map((header) => {
+          const value = row[header];
+          const type = typeof value === 'number' ? 'Number' : 'String';
+          return `        <Cell><Data ss:Type="${type}">${escapeXml(
+            String(value ?? ''),
+          )}</Data></Cell>`;
+        })
+        .join('\n')}\n      </Row>`,
+    );
   }
 
-  xml += '    </Table>\n';
-  xml += '  </Worksheet>\n';
-  xml += '</Workbook>';
+  const xml =
+    '<?xml version="1.0"?>\n' +
+    '<Workbook xmlns="urn:schemas-microsoft-com:office:spreadsheet" ' +
+    'xmlns:ss="urn:schemas-microsoft-com:office:spreadsheet">\n' +
+    '  <Worksheet ss:Name="Sheet1">\n' +
+    '    <Table>\n' +
+    '      <Row>\n' +
+    `        ${headerRow}\n` +
+    '      </Row>\n' +
+    rowChunks.join('\n') +
+    '\n    </Table>\n' +
+    '  </Worksheet>\n' +
+    '</Workbook>';
 
   return new Blob([xml], { type: 'application/vnd.ms-excel' });
 }
 
 async function exportToPDF(data: ExportData, title: string): Promise<Blob> {
   const { headers, rows } = data;
 
-  let html = `<!DOCTYPE html>
+  const headPart =
+    `<!DOCTYPE html>
 <html>
 <head>
   <meta charset="utf-8">
@@ -172,49 +174,32 @@ async function exportToPDF(data: ExportData, title: string): Promise<Blob> {
   <table>
     <thead>
       <tr>
-`;
-
-  for (const header of headers) {
-    html += `        <th>${escapeHtml(header)}</th>\n`;
-  }
-
-  html += `      </tr>
+` +
+    headers.map((header) => `        <th>${escapeHtml(header)}</th>`).join('\n') +
+    `
+      </tr>
     </thead>
     <tbody>
 `;
 
+  const bodyChunks: string[] = [];
   for (const row of rows) {
-    html += '      <tr>\n';
-    for (const header of headers) {
-      html += `        <td>${escapeHtml(String(row[header] ?? ''))}</td>\n`;
-    }
-    html += '      </tr>\n';
+    bodyChunks.push(
+      '      <tr>\n' +
+        headers
+          .map((header) => `        <td>${escapeHtml(String(row[header] ?? ''))}</td>`)
+          .join('\n') +
+        '\n      </tr>',
+    );
   }
 
-  html += `    </tbody>
+  const tailPart = `
+    </tbody>
   </table>
 </body>
 </html>`;
 
-  return new Blob([html], { type: 'text/html' });
-}
-
-function escapeXml(str: string): string {
-  return str
-    .replace(/&/g, '&amp;')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&apos;');
-}
-
-function escapeHtml(str: string): string {
-  return str
-    .replace(/&/g, '&amp;')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&#039;');
+  return new Blob([headPart, bodyChunks.join('\n'), tailPart], { type: 'text/html' });
 }
 
 function extensionForFormat(format: ExportFormat): string {

src/lib/export/utils.test.ts

@@ -1,5 +1,12 @@
 import { describe, expect, it } from 'vitest';
-import { defaultSort, normalizeFilters, prepareExportData } from './utils';
+import {
+  createCSVSnapshot,
+  createJSONSnapshot,
+  defaultSort,
+  escapeCSVCell,
+  normalizeFilters,
+  prepareExportData,
+} from './utils';
 
 describe('export utilities', () => {
   const dataset = {
@@ -33,4 +40,78 @@ describe('export utilities', () => {
       { field: 'createdDate', direction: 'desc' },
     ]);
   });
+
+  it('escapes CSV cells containing delimiters, quotes, and newlines', () => {
+    expect(escapeCSVCell('plain')).toBe('plain');
+    expect(escapeCSVCell('a,b')).toBe('"a,b"');
+    expect(escapeCSVCell('say "hi"')).toBe('"say ""hi"""');
+    expect(escapeCSVCell('line\nbreak')).toBe('"line\nbreak"');
+    expect(escapeCSVCell(null)).toBe('');
+    expect(escapeCSVCell(undefined)).toBe('');
+    expect(escapeCSVCell(0)).toBe('0');
+  });
+
+  it('streams CSV in bounded chunks with a leading header row', async () => {
+    const chunks: string[][] = [];
+    for await (const chunk of createCSVSnapshot(dataset, { chunkSize: 2 })) {
+      chunks.push(chunk);
+    }
+
+    const lines = chunks.flat();
+    expect(lines[0]).toBe('id,name,status,date,value');
+    expect(lines).toHaveLength(4);
+    expect(lines[1]).toContain('Gamma');
+    expect(lines[3]).toContain('Beta');
+    expect(chunks).toHaveLength(3);
+    expect(chunks[1]).toHaveLength(2);
+    expect(chunks[2]).toHaveLength(1);
+  });
+
+  it('emits a header-only CSV when there are no rows', async () => {
+    const chunks: string[][] = [];
+    for await (const chunk of createCSVSnapshot({ headers: ['a'], rows: [] })) {
+      chunks.push(chunk);
+    }
+
+    const lines = chunks.flat();
+    expect(lines).toEqual(['a']);
+  });
+
+  it('reports per-chunk progress via onChunk callback', async () => {
+    const seen: Array<{ index: number; total: number }> = [];
+    for await (const _chunk of createCSVSnapshot(dataset, {
+      chunkSize: 2,
+      onChunk: (index, total) => seen.push({ index, total }),
+    })) {
+      // drain generator
+    }
+
+    expect(seen).toEqual([
+      { index: 2, total: 3 },
+      { index: 3, total: 3 },
+    ]);
+  });
+
+  it('streams JSON row chunks that join into a valid, pretty-printed array', async () => {
+    const parts: string[] = [];
+    for await (const part of createJSONSnapshot(dataset, { chunkSize: 2 })) {
+      parts.push(part);
+    }
+
+    const payload = parts.join('');
+    const parsed = JSON.parse(payload);
+    expect(parsed).toHaveLength(3);
+    expect(parsed[0].name).toBe('Gamma');
+    expect(parsed[2].name).toBe('Beta');
+    expect(payload).toBe(JSON.stringify(dataset.rows, null, 2));
+  });
+
+  it('emits an empty JSON array when there are no rows', async () => {
+    const parts: string[] = [];
+    for await (const part of createJSONSnapshot({ headers: ['a'], rows: [] })) {
+      parts.push(part);
+    }
+
+    expect(parts.join('')).toBe('[]');
+  });
 });