diff --git a/src/lib/export-scheduler/__tests__/exporter.test.ts b/src/lib/export-scheduler/__tests__/exporter.test.ts index 35adc54a..9420e2da 100644 --- a/src/lib/export-scheduler/__tests__/exporter.test.ts +++ b/src/lib/export-scheduler/__tests__/exporter.test.ts @@ -6,6 +6,15 @@ import { describe, it, expect } from 'vitest'; import { exportData, fetchDataForTemplate } from '../exporter'; import { ExportTemplate } from '../types'; +function readBlobText(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result)); + reader.onerror = () => reject(reader.error); + reader.readAsText(blob); + }); +} + describe('Data Exporter', () => { const mockTemplate: ExportTemplate = { id: 'test-1', @@ -31,14 +40,23 @@ describe('Data Exporter', () => { expect(result.blob).toBeInstanceOf(Blob); expect(result.fileName).toContain('.csv'); expect(result.blob.type).toContain('text/csv'); + + const text = await readBlobText(result.blob); + expect(text.split('\n')[0]).toBe('id,name,value'); + expect(text).toContain('Item 2'); + expect(text).toContain('Item 1'); }); - it('should export to JSON', async () => { + it('should export to JSON as an equivalent, valid array', async () => { const template = { ...mockTemplate, format: 'json' as const }; const result = await exportData(template, mockData); expect(result.blob).toBeInstanceOf(Blob); expect(result.fileName).toContain('.json'); expect(result.blob.type).toContain('application/json'); + + const text = await readBlobText(result.blob); + expect(JSON.parse(text)).toEqual(mockData.rows); + expect(text).toBe(JSON.stringify(mockData.rows, null, 2)); }); it('should export to XLSX', async () => { diff --git a/src/lib/export-scheduler/exporter.ts b/src/lib/export-scheduler/exporter.ts index a011425b..d1dba365 100644 --- a/src/lib/export-scheduler/exporter.ts +++ b/src/lib/export-scheduler/exporter.ts @@ -5,7 +5,15 @@ import { createLogger } from '@/lib/logging'; import { createCounterMetric, measureAsync } from '@/lib/logging/performance'; -import { ExportExecutionOptions, emitProgress, prepareExportData } from '@/lib/export'; +import { + createCSVSnapshot, + createJSONSnapshot, + escapeHtml, + escapeXml, + ExportExecutionOptions, + emitProgress, + prepareExportData, +} from '@/lib/export'; import { ExportFormat, ExportTemplate } from './types'; export interface ExportData { @@ -87,64 +95,57 @@ export async function exportData( } async function exportToCSV(data: ExportData): Promise { - const { headers, rows } = data; - - const escape = (value: unknown): string => { - const str = String(value ?? ''); - if (str.includes(',') || str.includes('"') || str.includes('\n')) { - return `"${str.replace(/"/g, '""')}"`; - } - return str; - }; - - const csvLines: string[] = []; - csvLines.push(headers.map(escape).join(',')); - - for (const row of rows) { - const values = headers.map((header) => escape(row[header])); - csvLines.push(values.join(',')); + const parts: string[] = []; + for await (const chunk of createCSVSnapshot(data)) { + parts.push(chunk.join('\n')); } - return new Blob([csvLines.join('\n')], { type: 'text/csv;charset=utf-8;' }); + return new Blob([parts.join('\n')], { type: 'text/csv;charset=utf-8;' }); } async function exportToJSON(data: ExportData): Promise { - return new Blob([JSON.stringify(data.rows, null, 2)], { - type: 'application/json;charset=utf-8;', - }); + const parts: string[] = []; + for await (const chunk of createJSONSnapshot(data)) { + parts.push(chunk); + } + + return new Blob(parts, { type: 'application/json;charset=utf-8;' }); } async function exportToXLSX(data: ExportData): Promise { const { headers, rows } = data; - let xml = '\n'; - xml += '${escapeXml(header)}\n`; - } - - xml += ' \n'; - + const headerRow = headers + .map((header) => `${escapeXml(header)}`) + .join('\n '); + const rowChunks: string[] = []; for (const row of rows) { - xml += ' \n'; - for (const header of headers) { - const value = row[header]; - const type = typeof value === 'number' ? 'Number' : 'String'; - xml += ` ${escapeXml( - String(value ?? ''), - )}\n`; - } - xml += ' \n'; + rowChunks.push( + ` \n${headers + .map((header) => { + const value = row[header]; + const type = typeof value === 'number' ? 'Number' : 'String'; + return ` ${escapeXml( + String(value ?? ''), + )}`; + }) + .join('\n')}\n `, + ); } - xml += ' \n'; - xml += ' \n'; - xml += ''; + const xml = + '\n' + + '\n' + + ' \n' + + ' \n' + + ' \n' + + ` ${headerRow}\n` + + ' \n' + + rowChunks.join('\n') + + '\n
\n' + + '
\n' + + '
'; return new Blob([xml], { type: 'application/vnd.ms-excel' }); } @@ -152,7 +153,8 @@ async function exportToXLSX(data: ExportData): Promise { async function exportToPDF(data: ExportData, title: string): Promise { const { headers, rows } = data; - let html = ` + const headPart = + ` @@ -172,49 +174,32 @@ async function exportToPDF(data: ExportData, title: string): Promise { -`; - - for (const header of headers) { - html += ` \n`; - } - - html += ` +` + + headers.map((header) => ` `).join('\n') + + ` + `; + const bodyChunks: string[] = []; for (const row of rows) { - html += ' \n'; - for (const header of headers) { - html += ` \n`; - } - html += ' \n'; + bodyChunks.push( + ' \n' + + headers + .map((header) => ` `) + .join('\n') + + '\n ', + ); } - html += ` + const tailPart = ` +
${escapeHtml(header)}
${escapeHtml(header)}
${escapeHtml(String(row[header] ?? ''))}
${escapeHtml(String(row[header] ?? ''))}
`; - return new Blob([html], { type: 'text/html' }); -} - -function escapeXml(str: string): string { - return str - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); -} - -function escapeHtml(str: string): string { - return str - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); + return new Blob([headPart, bodyChunks.join('\n'), tailPart], { type: 'text/html' }); } function extensionForFormat(format: ExportFormat): string { diff --git a/src/lib/export/utils.test.ts b/src/lib/export/utils.test.ts index 2e1aea64..52809a49 100644 --- a/src/lib/export/utils.test.ts +++ b/src/lib/export/utils.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from 'vitest'; -import { defaultSort, normalizeFilters, prepareExportData } from './utils'; +import { + createCSVSnapshot, + createJSONSnapshot, + defaultSort, + escapeCSVCell, + normalizeFilters, + prepareExportData, +} from './utils'; describe('export utilities', () => { const dataset = { @@ -33,4 +40,78 @@ describe('export utilities', () => { { field: 'createdDate', direction: 'desc' }, ]); }); + + it('escapes CSV cells containing delimiters, quotes, and newlines', () => { + expect(escapeCSVCell('plain')).toBe('plain'); + expect(escapeCSVCell('a,b')).toBe('"a,b"'); + expect(escapeCSVCell('say "hi"')).toBe('"say ""hi"""'); + expect(escapeCSVCell('line\nbreak')).toBe('"line\nbreak"'); + expect(escapeCSVCell(null)).toBe(''); + expect(escapeCSVCell(undefined)).toBe(''); + expect(escapeCSVCell(0)).toBe('0'); + }); + + it('streams CSV in bounded chunks with a leading header row', async () => { + const chunks: string[][] = []; + for await (const chunk of createCSVSnapshot(dataset, { chunkSize: 2 })) { + chunks.push(chunk); + } + + const lines = chunks.flat(); + expect(lines[0]).toBe('id,name,status,date,value'); + expect(lines).toHaveLength(4); + expect(lines[1]).toContain('Gamma'); + expect(lines[3]).toContain('Beta'); + expect(chunks).toHaveLength(3); + expect(chunks[1]).toHaveLength(2); + expect(chunks[2]).toHaveLength(1); + }); + + it('emits a header-only CSV when there are no rows', async () => { + const chunks: string[][] = []; + for await (const chunk of createCSVSnapshot({ headers: ['a'], rows: [] })) { + chunks.push(chunk); + } + + const lines = chunks.flat(); + expect(lines).toEqual(['a']); + }); + + it('reports per-chunk progress via onChunk callback', async () => { + const seen: Array<{ index: number; total: number }> = []; + for await (const _chunk of createCSVSnapshot(dataset, { + chunkSize: 2, + onChunk: (index, total) => seen.push({ index, total }), + })) { + // drain generator + } + + expect(seen).toEqual([ + { index: 2, total: 3 }, + { index: 3, total: 3 }, + ]); + }); + + it('streams JSON row chunks that join into a valid, pretty-printed array', async () => { + const parts: string[] = []; + for await (const part of createJSONSnapshot(dataset, { chunkSize: 2 })) { + parts.push(part); + } + + const payload = parts.join(''); + const parsed = JSON.parse(payload); + expect(parsed).toHaveLength(3); + expect(parsed[0].name).toBe('Gamma'); + expect(parsed[2].name).toBe('Beta'); + expect(payload).toBe(JSON.stringify(dataset.rows, null, 2)); + }); + + it('emits an empty JSON array when there are no rows', async () => { + const parts: string[] = []; + for await (const part of createJSONSnapshot({ headers: ['a'], rows: [] })) { + parts.push(part); + } + + expect(parts.join('')).toBe('[]'); + }); }); diff --git a/src/lib/export/utils.ts b/src/lib/export/utils.ts index a551870d..d85ac29e 100644 --- a/src/lib/export/utils.ts +++ b/src/lib/export/utils.ts @@ -113,3 +113,99 @@ export function defaultSort(columns?: string[]): ExportSort[] { return [{ field: columns[0], direction: 'asc' }]; } + +export interface ExportStreamOptions { + /** Max rows per yielded chunk. Keeps memory per chunk bounded on large datasets. */ + chunkSize?: number; + /** Optional hook invoked after each chunk is produced (e.g. to surface progress). */ + onChunk?: (index: number, total: number, chunk: string[]) => void; +} + +export function escapeCSVCell(value: unknown): string { + const str = String(value ?? ''); + if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) { + return `"${str.replace(/"/g, '""')}"`; + } + return str; +} + +export function escapeXml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +export function escapeHtml(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +/** + * Yield a CSV payload in bounded chunks instead of building the whole + * string in memory. Each chunk contains a batch of `chunkSize` rows. + */ +export async function* createCSVSnapshot( + data: ExportDataset, + options: ExportStreamOptions = {}, +): AsyncGenerator { + const { chunkSize = 500, onChunk } = options; + const { headers, rows } = data; + + const headerChunk = [headers.map(escapeCSVCell).join(',')]; + yield headerChunk; + + for (let index = 0; index < rows.length; index += chunkSize) { + const batch = rows.slice(index, index + chunkSize); + const chunk: string[] = []; + for (const row of batch) { + chunk.push(headers.map((header) => escapeCSVCell(row[header])).join(',')); + } + onChunk?.(index + batch.length, rows.length, chunk); + yield chunk; + } +} + +/** + * Yield a JSON array payload in bounded chunks. The wrapper `[` and `]` + * delimiters are emitted separately so memory stays bounded to `chunkSize` + * rows on large datasets. + */ +export async function* createJSONSnapshot( + data: ExportDataset, + options: ExportStreamOptions = {}, +): AsyncGenerator { + const { chunkSize = 500, onChunk } = options; + const { rows } = data; + + if (rows.length === 0) { + yield '[]'; + return; + } + + const indent = (value: unknown): string => + JSON.stringify(value, null, 2) + .split('\n') + .map((line) => ` ${line}`) + .join('\n'); + + yield '[\n'; + for (let index = 0; index < rows.length; index += chunkSize) { + const batch = rows.slice(index, index + chunkSize); + const chunk: string[] = []; + for (let rowIndex = 0; rowIndex < batch.length; rowIndex += 1) { + const globalIndex = index + rowIndex; + const isLast = globalIndex === rows.length - 1; + chunk.push(`${indent(batch[rowIndex])}${isLast ? '' : ','}\n`); + } + onChunk?.(index + batch.length, rows.length, chunk); + yield chunk.join(''); + } + yield ']'; +}