Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 13 additions & 6 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -246,9 +246,9 @@ Follow existing patterns in `mg-fs-utils` or `mg-tinynews`.

## HTML & XML Parsing

**Use `@tryghost/mg-utils` for all HTML and XML parsing. Do not use `cheerio`.**
**Use `@tryghost/mg-utils` for all HTML and XML parsing. Do not use `cheerio` or `jsdom`.**

See the [`mg-utils` README](packages/mg-utils/README.md) for full API documentation.
Powered by [linkedom](https://github.com/WebReflection/linkedom) — lightweight and memory-efficient. See the [`mg-utils` README](packages/mg-utils/README.md) for full API documentation.

```javascript
import {xmlUtils, domUtils} from '@tryghost/mg-utils';
Expand All @@ -258,10 +258,17 @@ const parsed = await xmlUtils.parseXml(xmlString);
const channel = parsed.rss.channel;
const items = [].concat(channel.item || []); // normalize single/array

// HTML: parse, manipulate, serialize
const frag = domUtils.parseFragment(html);
frag.$('.unwanted').forEach(el => el.remove());
const output = frag.html();
// HTML: use processFragment for automatic cleanup
const output = domUtils.processFragment(html, (frag) => {
frag.$('.unwanted').forEach(el => el.remove());
return frag.html();
});

// Async version when the callback needs to await
const output = await domUtils.processFragmentAsync(html, async (frag) => {
// ... async operations ...
return frag.html();
});
```

## Error Handling
Expand Down
10 changes: 7 additions & 3 deletions packages/mg-blogger/lib/process.js
Original file line number Diff line number Diff line change
Expand Up @@ -360,9 +360,13 @@ const processPosts = async (posts, options) => {
// Filter out falsy items in the post list
posts = posts.filter(i => i);

return Promise.all(posts.map((post) => {
return processPost(post, options);
}));
const results = [];

for (let i = 0; i < posts.length; i++) {
results.push(await processPost(posts[i], options));
}

return results;
};

const all = async (input, {options}) => {
Expand Down
9 changes: 8 additions & 1 deletion packages/mg-chorus/lib/processor.js
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,14 @@ const processPost = (data, options) => {
};

const processPosts = (posts, options) => {
return posts.map(post => processPost(post, options));
const results = [];
for (let i = 0; i < posts.length; i++) {
const post = posts[i];
if (post) {
results.push(processPost(post, options));
}
}
return results;
};

const all = ({result, options}) => {
Expand Down
8 changes: 7 additions & 1 deletion packages/mg-curated-export/lib/process.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,13 @@ export default (input, ctx) => {
});

if (input.posts && input.posts.length > 0) {
output.posts = input.posts.map(post => processPost(post.json, globalUser, tags, ctx));
output.posts = [];
for (let i = 0; i < input.posts.length; i++) {
const post = input.posts[i];
if (post) {
output.posts.push(processPost(post.json, globalUser, tags, ctx));
}
}
}

return output;
Expand Down
9 changes: 8 additions & 1 deletion packages/mg-ghost-api/lib/processor.js
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,14 @@ const processPost = (ghPost) => {
};

const processPosts = (posts) => {
return posts.map(post => processPost(post));
const results = [];
for (let i = 0; i < posts.length; i++) {
const post = posts[i];
if (post) {
results.push(processPost(post));
}
}
return results;
};

const processAuthor = (ghAuthor) => {
Expand Down
8 changes: 7 additions & 1 deletion packages/mg-jekyll-export/lib/process.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,13 @@ export default (input, options = {}) => {
};

if (input.posts && input.posts.length > 0) {
output.posts = input.posts.map(post => processPost(post.fileName, post.fileContents, globalUser, options));
output.posts = [];
for (let i = 0; i < input.posts.length; i++) {
const post = input.posts[i];
if (post) {
output.posts.push(processPost(post.fileName, post.fileContents, globalUser, options));
}
}
}

return output;
Expand Down
9 changes: 8 additions & 1 deletion packages/mg-letterdrop/lib/processor.js
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,14 @@ const processPost = (data, options) => {
};

const processPosts = (posts, options) => {
return posts.map(post => processPost(post, options));
const results = [];
for (let i = 0; i < posts.length; i++) {
const post = posts[i];
if (post) {
results.push(processPost(post, options));
}
}
return results;
};

const all = ({result, options}) => {
Expand Down
13 changes: 8 additions & 5 deletions packages/mg-letterdrop/test/processor.test.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
import assert from 'node:assert/strict';
import {readFileSync} from 'node:fs';
import {dirname, join} from 'node:path';
import {describe, it} from 'node:test';
import {createRequire} from 'node:module';
import {fileURLToPath} from 'node:url';
import processor from '../lib/processor.js';

const require = createRequire(import.meta.url);
const fixture = require('./fixtures/api-response.json');
const __dirname = dirname(fileURLToPath(import.meta.url));
const fixtureData = JSON.parse(readFileSync(join(__dirname, 'fixtures/api-response.json'), 'utf8'));
const fixture = () => structuredClone(fixtureData);

describe('Process', function () {
it('Can convert a single post', function () {
const ctx = {
result: fixture,
result: fixture(),
options: {
url: 'https://example.com',
addPrimaryTag: 'Newsletter',
Expand Down Expand Up @@ -68,7 +71,7 @@ describe('Process', function () {

it('Converts signup iframes to Portal links', function () {
const ctx = {
result: fixture,
result: fixture(),
options: {
url: 'https://example.com',
addPrimaryTag: 'Newsletter',
Expand Down
9 changes: 8 additions & 1 deletion packages/mg-libsyn/lib/processor.js
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,14 @@ const processPost = (libsynPost, author, tags, options, errors) => { // eslint-d
return post;
};
const processPosts = (posts, author, tags, options, errors) => { // eslint-disable-line no-shadow
return posts.map(post => processPost(post, author, tags, options, errors));
const results = [];
for (let i = 0; i < posts.length; i++) {
const post = posts[i];
if (post) {
results.push(processPost(post, author, tags, options, errors));
}
}
return results;
};

const all = ({result, errors, options}) => { // eslint-disable-line no-shadow
Expand Down
25 changes: 14 additions & 11 deletions packages/mg-libsyn/test/processor.test.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import assert from 'node:assert/strict';
import {readFileSync} from 'node:fs';
import {dirname, join} from 'node:path';
import {describe, it} from 'node:test';
import {createRequire} from 'node:module';
import {fileURLToPath} from 'node:url';
import processor from '../lib/processor.js';

const require = createRequire(import.meta.url);
const fixture = require('./fixtures/feed.json');
const __dirname = dirname(fileURLToPath(import.meta.url));
const fixtureData = JSON.parse(readFileSync(join(__dirname, 'fixtures/feed.json'), 'utf8'));
const fixture = () => structuredClone(fixtureData);

describe('durationToSeconds', function () {
it('Minutes with no seconds', function () {
Expand Down Expand Up @@ -41,7 +44,7 @@ describe('durationToSeconds', function () {
describe('Process posts', function () {
it('Can process posts', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
author: {
name: 'Test Author',
slug: 'test-author',
Expand All @@ -61,7 +64,7 @@ describe('Process posts', function () {

it('Post has required fields', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
author: {
name: 'Test Author',
slug: 'test-author',
Expand Down Expand Up @@ -118,7 +121,7 @@ describe('Process posts', function () {

it('Can add a tag', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
author: {
name: 'Test Author',
slug: 'test-author',
Expand Down Expand Up @@ -152,7 +155,7 @@ describe('Process posts', function () {

it('Can use feed categories', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
tags: ['Lorem', 'Ipsum', 'dolor'],
author: {
name: 'Test Author',
Expand Down Expand Up @@ -200,7 +203,7 @@ describe('Process posts', function () {

it('Can use item categories', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
author: {
name: 'Test Author',
slug: 'test-author',
Expand Down Expand Up @@ -245,7 +248,7 @@ describe('Process posts', function () {
describe('Process content', function () {
it('Remove empty p tags', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
author: {
name: 'Test Author',
slug: 'test-author',
Expand All @@ -268,7 +271,7 @@ describe('Process content', function () {

it('Use Libsyn embeds', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
author: {
name: 'Test Author',
slug: 'test-author',
Expand All @@ -293,7 +296,7 @@ describe('Process content', function () {

it('Use Audio cards', function () {
const data = {
posts: fixture.rss.channel.item,
posts: fixture().rss.channel.item,
author: {
name: 'Test Author',
slug: 'test-author',
Expand Down
28 changes: 14 additions & 14 deletions packages/mg-linkfixer/lib/LinkFixer.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import {join} from 'node:path';
import _ from 'lodash';
import {domUtils} from '@tryghost/mg-utils';

const {parseFragment} = domUtils;
const {processFragment} = domUtils;

// @TODO: expand this list
const htmlFields = ['html'];
Expand Down Expand Up @@ -130,24 +130,24 @@ export default class LinkFixer {
}

async processHTML(html) {
const parsed = parseFragment(html);
return processFragment(html, (parsed) => {
for (const el of parsed.$('a')) {
let href = el.getAttribute('href');

for (const el of parsed.$('a')) {
let href = el.getAttribute('href');
if (!href) {
continue;
}

if (!href) {
continue;
}

// Clean the URL, matching the links stored in the linkMap
let updatedURL = this.cleanURL(href);
// Clean the URL, matching the links stored in the linkMap
let updatedURL = this.cleanURL(href);

if (this.linkMap[updatedURL]) {
el.setAttribute('href', this.linkMap[updatedURL]);
if (this.linkMap[updatedURL]) {
el.setAttribute('href', this.linkMap[updatedURL]);
}
}
}

return parsed.html();
return parsed.html();
});
}

async processLexical(lexical) {
Expand Down
Loading