feat: add invisible characters detection, plugin and style

src/parser/InvisibleCharactersPlugin.ts

@@ -0,0 +1,101 @@
+import {
+  Decoration,
+  DecorationSet,
+  EditorView,
+  hoverTooltip,
+  WidgetType,
+} from "@codemirror/view";
+import { EditorState, Extension, RangeSetBuilder, StateField } from "@codemirror/state";
+
+import {
+  findInvisibleCharacters,
+  FoundInvisibleChar,
+  InvisibleChar,
+} from "./invisibleCharacters/findInvisibleCharacters";
+
+const nonBreakingSpaceDecoration = Decoration.mark({
+  attributes: { class: "cm-invisible-char-nbsp" },
+});
+
+class ZeroWidthWidget extends WidgetType {
+  eq(): boolean {
+    return true;
+  }
+
+  toDOM(): HTMLElement {
+    const bar = document.createElement("span");
+    bar.className = "cm-invisible-char-zero-width";
+    return bar;
+  }
+}
+
+const zeroWidthDecoration = Decoration.widget({
+  widget: new ZeroWidthWidget(),
+  side: 1,
+});
+
+type InvisibleCharactersState = {
+  found: FoundInvisibleChar[];
+  decorations: DecorationSet;
+};
+
+function buildState(state: EditorState): InvisibleCharactersState {
+  const found = findInvisibleCharacters(state.doc.toString());
+  const builder = new RangeSetBuilder<Decoration>();
+  found.forEach(({ index, char }) => {
+    if (char.kind === "zeroWidth") {
+      builder.add(index, index, zeroWidthDecoration);
+    } else {
+      builder.add(index, index + char.value.length, nonBreakingSpaceDecoration);
+    }
+  });
+  return { found, decorations: builder.finish() };
+}
+
+const invisibleCharactersField = StateField.define<InvisibleCharactersState>({
+  create: buildState,
+  update(value, tr) {
+    return tr.docChanged ? buildState(tr.state) : value;
+  },
+  provide: (field) =>
+    EditorView.decorations.from(field, (value) => value.decorations),
+});
+
+export const InvisibleCharactersPlugin = (): Extension[] => [
+  invisibleCharactersField,
+];
+
+/**
+ * Hover tooltip naming the invisible character under the cursor.
+ *
+ * `getLabel` is supplied by the consuming app so the label can be localized
+ * there; this package holds no translations.
+ */
+export const invisibleCharactersTooltip = (
+  getLabel: (char: InvisibleChar) => string
+): Extension =>
+  hoverTooltip((view, pos, side) => {
+    const found = view.state
+      .field(invisibleCharactersField, false)
+      ?.found.find(({ index, char }) =>
+        side < 0
+          ? pos > index && pos <= index + char.value.length
+          : pos >= index && pos < index + char.value.length
+      );
+
+    if (!found) {
+      return null;
+    }
+
+    return {
+      pos: found.index,
+      end: found.index + found.char.value.length,
+      above: true,
+      create: () => {
+        const dom = document.createElement("div");
+        dom.className = "cm-invisible-char-tooltip";
+        dom.textContent = getLabel(found.char);
+        return { dom };
+      },
+    };
+  });

src/parser/invisibleCharacters/findInvisibleCharacters.test.ts

@@ -0,0 +1,83 @@
+import {
+  findInvisibleCharacters,
+  INVISIBLE_CHARACTERS,
+} from "./findInvisibleCharacters";
+
+describe("INVISIBLE_CHARACTERS", () => {
+  it("registers each character exactly once", () => {
+    const values = INVISIBLE_CHARACTERS.map(({ value }) => value);
+    expect(new Set(values).size).toBe(values.length);
+  });
+
+  it("registers only single code units", () => {
+    INVISIBLE_CHARACTERS.forEach(({ value }) => {
+      expect(value).toHaveLength(1);
+    });
+  });
+});
+
+describe("findInvisibleCharacters", () => {
+  it("returns nothing for an empty string", () => {
+    expect(findInvisibleCharacters("")).toEqual([]);
+  });
+
+  it("returns nothing for text with no invisible characters", () => {
+    expect(findInvisibleCharacters("a plain sentence")).toEqual([]);
+  });
+
+  it("does not match a regular space", () => {
+    expect(findInvisibleCharacters("a b c")).toEqual([]);
+  });
+
+  it.each([
+    ["\u00a0", "nonBreakingSpace"],
+    ["\u202f", "nonBreakingSpace"],
+    ["\u2007", "nonBreakingSpace"],
+    ["\u200b", "zeroWidth"],
+    ["\ufeff", "zeroWidth"],
+    ["\u00ad", "zeroWidth"],
+  ])("finds %j and reports kind %s", (value, kind) => {
+    expect(findInvisibleCharacters(`a${value}b`)).toEqual([
+      { index: 1, char: { value, kind } },
+    ]);
+  });
+
+  it("finds a character at the first index", () => {
+    expect(findInvisibleCharacters("\u00a0abc")).toEqual([
+      { index: 0, char: { value: "\u00a0", kind: "nonBreakingSpace" } },
+    ]);
+  });
+
+  it("finds a character at the last index", () => {
+    expect(findInvisibleCharacters("abc\u00a0")).toEqual([
+      { index: 3, char: { value: "\u00a0", kind: "nonBreakingSpace" } },
+    ]);
+  });
+
+  it("finds multiple characters in ascending order", () => {
+    expect(findInvisibleCharacters("a\u00a0b\u200bc")).toEqual([
+      { index: 1, char: { value: "\u00a0", kind: "nonBreakingSpace" } },
+      { index: 3, char: { value: "\u200b", kind: "zeroWidth" } },
+    ]);
+  });
+
+  it("finds adjacent characters", () => {
+    expect(findInvisibleCharacters("\u00a0\u00a0")).toEqual([
+      { index: 0, char: { value: "\u00a0", kind: "nonBreakingSpace" } },
+      { index: 1, char: { value: "\u00a0", kind: "nonBreakingSpace" } },
+    ]);
+  });
+
+  it("reports code-unit offsets in text containing surrogate pairs", () => {
+    expect(findInvisibleCharacters("\u{1f600}\u00a0")).toEqual([
+      { index: 2, char: { value: "\u00a0", kind: "nonBreakingSpace" } },
+    ]);
+  });
+
+  it("is stable across repeated calls", () => {
+    const text = "a\u00a0b";
+    expect(findInvisibleCharacters(text)).toEqual(
+      findInvisibleCharacters(text)
+    );
+  });
+});

src/parser/invisibleCharacters/findInvisibleCharacters.ts

@@ -0,0 +1,59 @@
+export type InvisibleCharKind = "nonBreakingSpace" | "zeroWidth";
+
+export type InvisibleChar = {
+  value: string;
+  kind: InvisibleCharKind;
+};
+
+export const INVISIBLE_CHARACTERS: InvisibleChar[] = [
+  { value: "\u00a0", kind: "nonBreakingSpace" },
+  { value: "\u202f", kind: "nonBreakingSpace" },
+  { value: "\u2007", kind: "nonBreakingSpace" },
+  { value: "\u200b", kind: "zeroWidth" },
+  { value: "\ufeff", kind: "zeroWidth" },
+  { value: "\u00ad", kind: "zeroWidth" },
+];
+
+const BY_VALUE = new Map(
+  INVISIBLE_CHARACTERS.map((char) => [char.value, char])
+);
+
+// Built from escapes rather than the raw characters so the regex source stays
+// readable — literal invisible characters here would be the very problem this
+// module exists to surface.
+const CHARACTER_CLASS = `[${INVISIBLE_CHARACTERS.map(({ value }) =>
+  `\\u${value.charCodeAt(0).toString(16).padStart(4, "0")}`
+).join("")}]`;
+
+const HAS_INVISIBLE = new RegExp(CHARACTER_CLASS);
+const ALL_INVISIBLE = new RegExp(CHARACTER_CLASS, "g");
+
+export type FoundInvisibleChar = {
+  index: number;
+  char: InvisibleChar;
+};
+
+/**
+ * Finds every invisible character in `text`.
+ *
+ * `index` is a UTF-16 code-unit offset, so it can be used directly with
+ * `String.prototype.substring` and with CodeMirror document positions.
+ * Results are ordered by ascending `index`.
+ */
+export function findInvisibleCharacters(text: string): FoundInvisibleChar[] {
+  if (!HAS_INVISIBLE.test(text)) {
+    return [];
+  }
+
+  const found: FoundInvisibleChar[] = [];
+  ALL_INVISIBLE.lastIndex = 0;
+  let match = ALL_INVISIBLE.exec(text);
+  while (match !== null) {
+    const char = BY_VALUE.get(match[0]);
+    if (char) {
+      found.push({ index: match.index, char });
+    }
+    match = ALL_INVISIBLE.exec(text);
+  }
+  return found;
+}

src/parser/invisibleCharactersStyle.ts

@@ -0,0 +1,39 @@
+import { StyledComponent } from "@emotion/styled";
+
+export type InvisibleCharacterColors = {
+  nonBreakingSpaceBackground: string;
+  zeroWidthMarker: string;
+};
+
+const DEFAULT_COLORS: InvisibleCharacterColors = {
+  nonBreakingSpaceBackground: "#a4d0ff",
+  zeroWidthMarker: "#ff9800",
+};
+
+type Props = {
+  styled: (component: any) => any;
+  colors?: InvisibleCharacterColors;
+  component?: any;
+};
+
+export const generateInvisibleCharactersStyle = ({
+  styled,
+  colors = DEFAULT_COLORS,
+  component = "div",
+}: Props): StyledComponent<any> => {
+  return styled(component)`
+    & .cm-invisible-char-nbsp {
+      background-color: ${colors.nonBreakingSpaceBackground};
+      border-radius: 2px;
+    }
+
+    & .cm-invisible-char-zero-width {
+      display: inline-block;
+      width: 2px;
+      height: 1em;
+      background-color: ${colors.zeroWidthMarker};
+      vertical-align: text-bottom;
+      border-radius: 1px;
+    }
+  `;
+};

src/tolgee-editor.ts

@@ -20,3 +20,17 @@ export {
   PO_MSGCTXT_KEY_SEPARATOR,
 } from "./parser/KeyNamePlugin";
 export { generateKeyNameStyle } from "./parser/keyNameStyle";
+export {
+  InvisibleCharactersPlugin,
+  invisibleCharactersTooltip,
+} from "./parser/InvisibleCharactersPlugin";
+export {
+  findInvisibleCharacters,
+  INVISIBLE_CHARACTERS,
+} from "./parser/invisibleCharacters/findInvisibleCharacters";
+export type {
+  InvisibleChar,
+  InvisibleCharKind,
+  FoundInvisibleChar,
+} from "./parser/invisibleCharacters/findInvisibleCharacters";
+export { generateInvisibleCharactersStyle } from "./parser/invisibleCharactersStyle";