`;
+ });
- // TODO: this should be a parser plugin
- // Wrap nested lists in HTML card
- parsed.$('ul li ul, ol li ol, ol li ul, ul li ol').forEach((nestedList) => {
- // Walk up to the nearest parent ul/ol (equivalent to parentsUntil('ul, ol').parent())
- let topList = nestedList.parentElement?.closest('ul, ol');
- if (topList) {
- domUtils.insertBefore(topList, '');
- domUtils.insertAfter(topList, '');
- }
- });
+ parsed.$('.sqs-video-wrapper').forEach((el) => {
+ const theHtml = decode(el.getAttribute('data-html'));
+ const embedWrapper = el.closest('.embed-block-wrapper');
+ const parent = embedWrapper ? embedWrapper.parentElement : null;
- // Convert HTML back to a string
- html = parsed.html();
+ if (parent) {
+ domUtils.replaceWith(parent, `${theHtml}`);
+ }
+ });
- // Trim whitespace
- html = html.trim();
+ // TODO: this should be a parser plugin
+ // Wrap nested lists in HTML card
+ parsed.$('ul li ul, ol li ol, ol li ul, ul li ol').forEach((nestedList) => {
+ // Walk up to the nearest parent ul/ol (equivalent to parentsUntil('ul, ol').parent())
+ let topList = nestedList.parentElement?.closest('ul, ol');
+ if (topList) {
+ domUtils.insertBefore(topList, '');
+ domUtils.insertAfter(topList, '');
+ }
+ });
- return html;
+ return parsed.html().trim();
+ });
};
// The feature images is not "connected" to the post, other than it's located
@@ -279,12 +273,17 @@ const processPost = (sqPost, index, items, users, options) => {
const processPosts = (items, users, options) => {
const postsOutput = [];
- items.forEach((sqPost, index) => {
- postsOutput.push(processPost(sqPost, index, items, users, options));
- });
+ for (let i = 0; i < items.length; i++) {
+ const item = items[i];
+ if (item) {
+ const result = processPost(item, i, items, users, options);
+ if (result) {
+ postsOutput.push(result);
+ }
+ }
+ }
- // don't return empty post objects
- return postsOutput.filter(post => post);
+ return postsOutput;
};
const processUsers = (authors) => {
diff --git a/packages/mg-substack/lib/process.js b/packages/mg-substack/lib/process.js
index bde40f430..7ce96b1ce 100644
--- a/packages/mg-substack/lib/process.js
+++ b/packages/mg-substack/lib/process.js
@@ -862,9 +862,13 @@ export default async (input, ctx) => {
}
if (input.posts && input.posts.length > 0) {
- output.posts = input.posts.map((post) => {
- return processPost(post, siteUrl, options);
- });
+ output.posts = [];
+ for (let i = 0; i < input.posts.length; i++) {
+ const post = input.posts[i];
+ if (post) {
+ output.posts.push(processPost(post, siteUrl, options));
+ }
+ }
}
return output;
diff --git a/packages/mg-utils/README.md b/packages/mg-utils/README.md
index 97eacc2f3..094c785a4 100644
--- a/packages/mg-utils/README.md
+++ b/packages/mg-utils/README.md
@@ -17,42 +17,80 @@ or
### DOM Utilities
-Parse and manipulate HTML fragments with proper HTML5 serialization:
+Lightweight HTML parsing and manipulation powered by [linkedom](https://github.com/WebReflection/linkedom). Use `processFragment` for most cases — it parses HTML, passes the fragment to your callback, and automatically cleans up:
```js
import {domUtils} from '@tryghost/mg-utils';
-const {
- parseFragment,
- serializeNode,
- serializeChildren,
- replaceWith,
- insertBefore,
- insertAfter,
- wrap,
- createElement,
- attr,
- is,
- parents,
- lastParent,
- setStyle,
- isComment,
- getCommentData
-} = domUtils;
-
-// Parse an HTML fragment
-const parsed = parseFragment('
Hello World
');
-
-// Query elements (returns array)
-const paragraphs = parsed.$('p');
-
-// Get serialized HTML
-const html = parsed.html();
-
-// Get text content
-const text = parsed.text();
+const {processFragment, processFragmentAsync} = domUtils;
+
+// Parse, manipulate, and get the result in one step
+const html = processFragment('
Hello
World
', (parsed) => {
+ for (const el of parsed.$('.remove')) {
+ el.remove();
+ }
+ return parsed.html();
+});
+// => '
Hello
'
+
+// Extract data from HTML
+const title = processFragment(rawHtml, parsed => parsed.$('h1')[0]?.textContent || '');
+
+// Async version for callbacks that need to await
+const result = await processFragmentAsync(html, async (parsed) => {
+ for (const img of parsed.$('img')) {
+ const newSrc = await processImage(img.getAttribute('src'));
+ img.setAttribute('src', newSrc);
+ }
+ return parsed.html();
+});
```
+The `parsed` fragment provides:
+- **`parsed.$(selector, context?)`** — query elements (returns `Element[]`)
+- **`parsed.html()`** — serialize the fragment back to an HTML string
+- **`parsed.text()`** — get text content
+- **`parsed.document`** — access the underlying `Document`
+- **`parsed.body`** — access the `` element
+
+For long-lived or complex processing where a callback doesn't fit, use `parseFragment` directly:
+
+```js
+const {parseFragment} = domUtils;
+
+const parsed = parseFragment(html);
+// ... extensive manipulation ...
+const result = parsed.html();
+```
+
+### DOM Manipulation Helpers
+
+```js
+const {replaceWith, insertBefore, insertAfter, wrap, createElement, attr} = domUtils;
+
+const parsed = parseFragment('
Old
');
+const p = parsed.$('p')[0];
+
+replaceWith(p, 'New'); // Replace element with HTML string or Node
+insertBefore(el, ''); // Insert before element
+insertAfter(el, ''); // Insert after element
+wrap(el, ''); // Wrap element in a new parent
+
+const div = createElement(parsed.document, 'div', {class: 'wrapper'});
+
+attr(el, 'href'); // Get attribute (returns '' if missing)
+attr(el, 'href', '/new-url'); // Set attribute
+```
+
+### Additional Element Utilities
+
+- **`is(el, selector)`** — check if element matches a CSS selector
+- **`parents(el, selector?)`** — get all parent elements, optionally filtered
+- **`lastParent(el, selector)`** — get the furthest parent matching selector
+- **`setStyle(el, property, value)`** — set a CSS style property
+- **`isComment(node)`** / **`getCommentData(node)`** — comment node helpers
+- **`serializeNode(node)`** / **`serializeChildren(node)`** — HTML5-compliant serialization
+
### XML Utilities
Parse XML strings or files into JavaScript objects using `fast-xml-parser`:
@@ -72,14 +110,6 @@ const data = await parseXml('/path/to/file.xml');
const data = await parseXml(xmlString, {attributeNamePrefix: ''});
```
-### Key Features
-
-- **HTML5-compliant serialization**: Void elements (``, ``, ` `) are self-closing, non-void elements (`');
+ assert.equal(processed, '
');
});
it('Can convert WP post embed', async function () {
diff --git a/packages/mg-wp-api/utils/users-html-to-json.js b/packages/mg-wp-api/utils/users-html-to-json.js
index 1f7b2ed48..af8f405a6 100644
--- a/packages/mg-wp-api/utils/users-html-to-json.js
+++ b/packages/mg-wp-api/utils/users-html-to-json.js
@@ -9,7 +9,7 @@ import {dirname, join} from 'node:path';
import {domUtils} from '@tryghost/mg-utils';
import {slugify} from '@tryghost/string';
-const {parseFragment} = domUtils;
+const {processFragment} = domUtils;
if (!process.argv[2]) {
console.error('Please provide a path to the file'); // eslint-disable-line no-console
@@ -21,45 +21,47 @@ const destPath = join(desitnationDir, 'users.json');
const html = readFileSync(process.argv[2], 'utf8');
-const parsed = parseFragment(html);
+const users = processFragment(html, (parsed) => {
+ let result = [];
-let users = [];
+ for (const el of parsed.$('tr[id^="user-"]')) {
+ const postsCell = el.querySelector('[data-colname="Posts"]');
+ const postCount = parseInt(postsCell ? postsCell.textContent.trim() : '0');
-for (const el of parsed.$('tr[id^="user-"]')) {
- const postsCell = el.querySelector('[data-colname="Posts"]');
- const postCount = parseInt(postsCell ? postsCell.textContent.trim() : '0');
+ if (postCount === 0) {
+ continue;
+ }
- if (postCount === 0) {
- continue;
- }
+ let id = parseInt(el.getAttribute('id').replace('user-', ''));
+ const emailCell = el.querySelector('[data-colname="Email"]');
+ let email = emailCell ? emailCell.textContent.trim() : '';
+ const nameCell = el.querySelector('[data-colname="Name"]');
+ let name = nameCell ? nameCell.textContent.trim() : '';
+ const usernameCell = el.querySelector('[data-colname="Username"]');
+ const usernameStrong = usernameCell ? usernameCell.querySelector('strong') : null;
+ let username = usernameStrong ? usernameStrong.textContent.trim() : '';
+ const usernameImg = usernameCell ? usernameCell.querySelector('img') : null;
+ let image = usernameImg ? usernameImg.getAttribute('src').trim().replace('s=64', 's=500') : '';
- let id = parseInt(el.getAttribute('id').replace('user-', ''));
- const emailCell = el.querySelector('[data-colname="Email"]');
- let email = emailCell ? emailCell.textContent.trim() : '';
- const nameCell = el.querySelector('[data-colname="Name"]');
- let name = nameCell ? nameCell.textContent.trim() : '';
- const usernameCell = el.querySelector('[data-colname="Username"]');
- const usernameStrong = usernameCell ? usernameCell.querySelector('strong') : null;
- let username = usernameStrong ? usernameStrong.textContent.trim() : '';
- const usernameImg = usernameCell ? usernameCell.querySelector('img') : null;
- let image = usernameImg ? usernameImg.getAttribute('src').trim().replace('s=64', 's=500') : '';
+ if (name.includes('—Unknown')) {
+ name = username;
+ }
- if (name.includes('—Unknown')) {
- name = username;
- }
+ let slug = slugify(name);
- let slug = slugify(name);
+ result.push({
+ id,
+ slug,
+ name,
+ email,
+ avatar_urls: {
+ 96: image
+ }
+ });
+ }
- users.push({
- id,
- slug,
- name,
- email,
- avatar_urls: {
- 96: image
- }
- });
-}
+ return result;
+});
writeFileSync(destPath, JSON.stringify(users, null, 4));
console.log(`✅ File saved to: ${destPath}`); // eslint-disable-line no-console
diff --git a/packages/mg-wp-xml/lib/process.js b/packages/mg-wp-xml/lib/process.js
index 1b8d77546..27f4c2aa4 100644
--- a/packages/mg-wp-xml/lib/process.js
+++ b/packages/mg-wp-xml/lib/process.js
@@ -19,7 +19,7 @@ const fixSerializedLengths = (value) => {
});
};
-const {parseFragment} = domUtils;
+const {processFragment} = domUtils;
const {getYouTubeID} = youtubeUtils;
// XML Parser configuration
@@ -239,17 +239,16 @@ const preProcessContent = async ({html, options}) => { // eslint-disable-line no
// Join the separated lines
html = splitIt.join('\n');
- const parsed = parseFragment(html);
-
- // Remove empty link elements, typically HTML anchors
- for (const el of parsed.$('a')) {
- if (el.innerHTML.length === 0) {
- el.remove();
+ html = processFragment(html, (parsed) => {
+ // Remove empty link elements, typically HTML anchors
+ for (const el of parsed.$('a')) {
+ if (el.innerHTML.length === 0) {
+ el.remove();
+ }
}
- }
- // convert HTML back to a string
- html = parsed.html();
+ return parsed.html();
+ });
// Convert shortcodes here to that they don't accidently get wrapped in