diff --git a/src/parser/InvisibleCharactersPlugin.ts b/src/parser/InvisibleCharactersPlugin.ts new file mode 100644 index 0000000..a796d68 --- /dev/null +++ b/src/parser/InvisibleCharactersPlugin.ts @@ -0,0 +1,101 @@ +import { + Decoration, + DecorationSet, + EditorView, + hoverTooltip, + WidgetType, +} from "@codemirror/view"; +import { EditorState, Extension, RangeSetBuilder, StateField } from "@codemirror/state"; + +import { + findInvisibleCharacters, + FoundInvisibleChar, + InvisibleChar, +} from "./invisibleCharacters/findInvisibleCharacters"; + +const nonBreakingSpaceDecoration = Decoration.mark({ + attributes: { class: "cm-invisible-char-nbsp" }, +}); + +class ZeroWidthWidget extends WidgetType { + eq(): boolean { + return true; + } + + toDOM(): HTMLElement { + const bar = document.createElement("span"); + bar.className = "cm-invisible-char-zero-width"; + return bar; + } +} + +const zeroWidthDecoration = Decoration.widget({ + widget: new ZeroWidthWidget(), + side: 1, +}); + +type InvisibleCharactersState = { + found: FoundInvisibleChar[]; + decorations: DecorationSet; +}; + +function buildState(state: EditorState): InvisibleCharactersState { + const found = findInvisibleCharacters(state.doc.toString()); + const builder = new RangeSetBuilder(); + found.forEach(({ index, char }) => { + if (char.kind === "zeroWidth") { + builder.add(index, index, zeroWidthDecoration); + } else { + builder.add(index, index + char.value.length, nonBreakingSpaceDecoration); + } + }); + return { found, decorations: builder.finish() }; +} + +const invisibleCharactersField = StateField.define({ + create: buildState, + update(value, tr) { + return tr.docChanged ? buildState(tr.state) : value; + }, + provide: (field) => + EditorView.decorations.from(field, (value) => value.decorations), +}); + +export const InvisibleCharactersPlugin = (): Extension[] => [ + invisibleCharactersField, +]; + +/** + * Hover tooltip naming the invisible character under the cursor. + * + * `getLabel` is supplied by the consuming app so the label can be localized + * there; this package holds no translations. + */ +export const invisibleCharactersTooltip = ( + getLabel: (char: InvisibleChar) => string +): Extension => + hoverTooltip((view, pos, side) => { + const found = view.state + .field(invisibleCharactersField, false) + ?.found.find(({ index, char }) => + side < 0 + ? pos > index && pos <= index + char.value.length + : pos >= index && pos < index + char.value.length + ); + + if (!found) { + return null; + } + + return { + pos: found.index, + end: found.index + found.char.value.length, + above: true, + create: () => { + const dom = document.createElement("div"); + dom.className = "cm-invisible-char-tooltip"; + dom.textContent = getLabel(found.char); + return { dom }; + }, + }; + }); diff --git a/src/parser/invisibleCharacters/findInvisibleCharacters.test.ts b/src/parser/invisibleCharacters/findInvisibleCharacters.test.ts new file mode 100644 index 0000000..61e786f --- /dev/null +++ b/src/parser/invisibleCharacters/findInvisibleCharacters.test.ts @@ -0,0 +1,83 @@ +import { + findInvisibleCharacters, + INVISIBLE_CHARACTERS, +} from "./findInvisibleCharacters"; + +describe("INVISIBLE_CHARACTERS", () => { + it("registers each character exactly once", () => { + const values = INVISIBLE_CHARACTERS.map(({ value }) => value); + expect(new Set(values).size).toBe(values.length); + }); + + it("registers only single code units", () => { + INVISIBLE_CHARACTERS.forEach(({ value }) => { + expect(value).toHaveLength(1); + }); + }); +}); + +describe("findInvisibleCharacters", () => { + it("returns nothing for an empty string", () => { + expect(findInvisibleCharacters("")).toEqual([]); + }); + + it("returns nothing for text with no invisible characters", () => { + expect(findInvisibleCharacters("a plain sentence")).toEqual([]); + }); + + it("does not match a regular space", () => { + expect(findInvisibleCharacters("a b c")).toEqual([]); + }); + + it.each([ + ["\u00a0", "nonBreakingSpace"], + ["\u202f", "nonBreakingSpace"], + ["\u2007", "nonBreakingSpace"], + ["\u200b", "zeroWidth"], + ["\ufeff", "zeroWidth"], + ["\u00ad", "zeroWidth"], + ])("finds %j and reports kind %s", (value, kind) => { + expect(findInvisibleCharacters(`a${value}b`)).toEqual([ + { index: 1, char: { value, kind } }, + ]); + }); + + it("finds a character at the first index", () => { + expect(findInvisibleCharacters("\u00a0abc")).toEqual([ + { index: 0, char: { value: "\u00a0", kind: "nonBreakingSpace" } }, + ]); + }); + + it("finds a character at the last index", () => { + expect(findInvisibleCharacters("abc\u00a0")).toEqual([ + { index: 3, char: { value: "\u00a0", kind: "nonBreakingSpace" } }, + ]); + }); + + it("finds multiple characters in ascending order", () => { + expect(findInvisibleCharacters("a\u00a0b\u200bc")).toEqual([ + { index: 1, char: { value: "\u00a0", kind: "nonBreakingSpace" } }, + { index: 3, char: { value: "\u200b", kind: "zeroWidth" } }, + ]); + }); + + it("finds adjacent characters", () => { + expect(findInvisibleCharacters("\u00a0\u00a0")).toEqual([ + { index: 0, char: { value: "\u00a0", kind: "nonBreakingSpace" } }, + { index: 1, char: { value: "\u00a0", kind: "nonBreakingSpace" } }, + ]); + }); + + it("reports code-unit offsets in text containing surrogate pairs", () => { + expect(findInvisibleCharacters("\u{1f600}\u00a0")).toEqual([ + { index: 2, char: { value: "\u00a0", kind: "nonBreakingSpace" } }, + ]); + }); + + it("is stable across repeated calls", () => { + const text = "a\u00a0b"; + expect(findInvisibleCharacters(text)).toEqual( + findInvisibleCharacters(text) + ); + }); +}); diff --git a/src/parser/invisibleCharacters/findInvisibleCharacters.ts b/src/parser/invisibleCharacters/findInvisibleCharacters.ts new file mode 100644 index 0000000..2a291aa --- /dev/null +++ b/src/parser/invisibleCharacters/findInvisibleCharacters.ts @@ -0,0 +1,59 @@ +export type InvisibleCharKind = "nonBreakingSpace" | "zeroWidth"; + +export type InvisibleChar = { + value: string; + kind: InvisibleCharKind; +}; + +export const INVISIBLE_CHARACTERS: InvisibleChar[] = [ + { value: "\u00a0", kind: "nonBreakingSpace" }, + { value: "\u202f", kind: "nonBreakingSpace" }, + { value: "\u2007", kind: "nonBreakingSpace" }, + { value: "\u200b", kind: "zeroWidth" }, + { value: "\ufeff", kind: "zeroWidth" }, + { value: "\u00ad", kind: "zeroWidth" }, +]; + +const BY_VALUE = new Map( + INVISIBLE_CHARACTERS.map((char) => [char.value, char]) +); + +// Built from escapes rather than the raw characters so the regex source stays +// readable — literal invisible characters here would be the very problem this +// module exists to surface. +const CHARACTER_CLASS = `[${INVISIBLE_CHARACTERS.map(({ value }) => + `\\u${value.charCodeAt(0).toString(16).padStart(4, "0")}` +).join("")}]`; + +const HAS_INVISIBLE = new RegExp(CHARACTER_CLASS); +const ALL_INVISIBLE = new RegExp(CHARACTER_CLASS, "g"); + +export type FoundInvisibleChar = { + index: number; + char: InvisibleChar; +}; + +/** + * Finds every invisible character in `text`. + * + * `index` is a UTF-16 code-unit offset, so it can be used directly with + * `String.prototype.substring` and with CodeMirror document positions. + * Results are ordered by ascending `index`. + */ +export function findInvisibleCharacters(text: string): FoundInvisibleChar[] { + if (!HAS_INVISIBLE.test(text)) { + return []; + } + + const found: FoundInvisibleChar[] = []; + ALL_INVISIBLE.lastIndex = 0; + let match = ALL_INVISIBLE.exec(text); + while (match !== null) { + const char = BY_VALUE.get(match[0]); + if (char) { + found.push({ index: match.index, char }); + } + match = ALL_INVISIBLE.exec(text); + } + return found; +} diff --git a/src/parser/invisibleCharactersStyle.ts b/src/parser/invisibleCharactersStyle.ts new file mode 100644 index 0000000..2079766 --- /dev/null +++ b/src/parser/invisibleCharactersStyle.ts @@ -0,0 +1,39 @@ +import { StyledComponent } from "@emotion/styled"; + +export type InvisibleCharacterColors = { + nonBreakingSpaceBackground: string; + zeroWidthMarker: string; +}; + +const DEFAULT_COLORS: InvisibleCharacterColors = { + nonBreakingSpaceBackground: "#a4d0ff", + zeroWidthMarker: "#ff9800", +}; + +type Props = { + styled: (component: any) => any; + colors?: InvisibleCharacterColors; + component?: any; +}; + +export const generateInvisibleCharactersStyle = ({ + styled, + colors = DEFAULT_COLORS, + component = "div", +}: Props): StyledComponent => { + return styled(component)` + & .cm-invisible-char-nbsp { + background-color: ${colors.nonBreakingSpaceBackground}; + border-radius: 2px; + } + + & .cm-invisible-char-zero-width { + display: inline-block; + width: 2px; + height: 1em; + background-color: ${colors.zeroWidthMarker}; + vertical-align: text-bottom; + border-radius: 1px; + } + `; +}; diff --git a/src/tolgee-editor.ts b/src/tolgee-editor.ts index 3b2c691..f72ced3 100644 --- a/src/tolgee-editor.ts +++ b/src/tolgee-editor.ts @@ -20,3 +20,17 @@ export { PO_MSGCTXT_KEY_SEPARATOR, } from "./parser/KeyNamePlugin"; export { generateKeyNameStyle } from "./parser/keyNameStyle"; +export { + InvisibleCharactersPlugin, + invisibleCharactersTooltip, +} from "./parser/InvisibleCharactersPlugin"; +export { + findInvisibleCharacters, + INVISIBLE_CHARACTERS, +} from "./parser/invisibleCharacters/findInvisibleCharacters"; +export type { + InvisibleChar, + InvisibleCharKind, + FoundInvisibleChar, +} from "./parser/invisibleCharacters/findInvisibleCharacters"; +export { generateInvisibleCharactersStyle } from "./parser/invisibleCharactersStyle";