From f733fcffccb0c27cf253deec6cfeb0f4c085ec0f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:30:41 -0400 Subject: [PATCH 01/24] OPENNLP-1894: Lattice segmentation over user-supplied mecab-format dictionaries A Viterbi decoder over word and connection costs segments languages written without spaces; the same engine serves Japanese and Korean because the language lives entirely in the dictionary. Unknown text is handled through the dictionary's character categories, and every span stays in original text coordinates. An installer fetches and unpacks a user-chosen dictionary archive at install time: nothing is bundled, no location is built in, and entry names are flattened so no archive path escapes the target directory. (cherry picked from commit a699c8aeaffd107a21e4ba8ed281582366d92bb8) --- .../tokenize/lattice/LatticeTokenizer.java | 266 ++++++++++++++ .../tokenize/lattice/MecabDictionary.java | 346 ++++++++++++++++++ .../lattice/MecabDictionaryInstaller.java | 226 ++++++++++++ .../tools/tokenize/lattice/Morpheme.java | 64 ++++ .../lattice/LatticeTokenizerTest.java | 153 ++++++++ .../lattice/MecabDictionaryInstallerTest.java | 133 +++++++ 6 files changed, 1188 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java new file mode 100644 index 0000000000..c022374a5e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.lattice.MecabDictionary.Category; +import opennlp.tools.tokenize.lattice.MecabDictionary.WordEntry; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Dictionary-driven segmentation for languages written without spaces: a Viterbi + * search over the word lattice of a {@link MecabDictionary}, minimizing the sum of + * word costs and connection costs. This is the segmentation approach behind Japanese + * and Korean morphological analysis; the same decoder serves both, since the language + * lives entirely in the user-supplied dictionary. + * + *

Unknown text is handled through the dictionary's character categories: where the + * lexicon has no entry, or a category always invokes them, unknown-word candidates are + * generated per category template, grouping runs of same-category characters when the + * category says so. Whitespace never joins a morpheme and is never reported as one. + * Every reported span is in original text coordinates.

+ * + *

{@link #analyze(String)} returns full morphemes with their dictionary features; + * the {@link Tokenizer} view reports just the surfaces and spans.

+ * + *

The tokenizer reads only immutable dictionary state and is safe to share between + * threads.

+ * + * @since 3.0.0 + */ +public class LatticeTokenizer implements Tokenizer { + + /** The context id of the beginning and end of text. */ + private static final int BOUNDARY_CONTEXT = 0; + + private final MecabDictionary dictionary; + + /** + * Initializes the tokenizer. + * + * @param dictionary The dictionary to segment with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public LatticeTokenizer(MecabDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + /** One lattice node: a candidate morpheme with its best path cost so far. */ + private static final class Node { + private final int start; + private final int end; + private final WordEntry entry; + private final boolean unknown; + private long pathCost = Long.MAX_VALUE; + private Node previous; + + private Node(int start, int end, WordEntry entry, boolean unknown) { + this.start = start; + this.end = end; + this.entry = entry; + this.unknown = unknown; + } + } + + /** + * Segments a text into morphemes with their dictionary features. + * + * @param text The text to segment. Must not be {@code null}. + * @return The morphemes in text order, spans in original coordinates, whitespace + * omitted. Never {@code null}; empty for empty or all-whitespace input. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + public List analyze(String text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + final List morphemes = new ArrayList<>(); + int start = 0; + while (start < text.length()) { + if (StringUtil.isWhitespace(text.charAt(start))) { + start++; + continue; + } + int end = start; + while (end < text.length() && !StringUtil.isWhitespace(text.charAt(end))) { + end++; + } + decode(text, start, end, morphemes); + start = end; + } + return morphemes; + } + + @Override + public String[] tokenize(String s) { + final List morphemes = analyze(s); + final String[] tokens = new String[morphemes.size()]; + for (int i = 0; i < tokens.length; i++) { + tokens[i] = morphemes.get(i).surface(); + } + return tokens; + } + + @Override + public Span[] tokenizePos(String s) { + final List morphemes = analyze(s); + final Span[] spans = new Span[morphemes.size()]; + for (int i = 0; i < spans.length; i++) { + spans[i] = morphemes.get(i).span(); + } + return spans; + } + + /** Runs the Viterbi search over one whitespace-free stretch of text. */ + private void decode(String text, int from, int to, List morphemes) { + final int length = to - from; + final List> endingAt = new ArrayList<>(length + 1); + for (int i = 0; i <= length; i++) { + endingAt.add(new ArrayList<>()); + } + final boolean[] reachable = new boolean[length + 1]; + reachable[0] = true; + + for (int i = 0; i < length; i++) { + if (!reachable[i]) { + continue; + } + final List candidates = candidates(text, from, to, i); + for (final Node candidate : candidates) { + relax(candidate, i == 0 ? null : endingAt.get(i)); + if (candidate.pathCost < Long.MAX_VALUE) { + endingAt.get(candidate.end - from).add(candidate); + reachable[candidate.end - from] = true; + } + } + } + + Node best = null; + for (final Node node : endingAt.get(length)) { + final long total = node.pathCost + + dictionary.connectionCost(node.entry.rightId(), BOUNDARY_CONTEXT); + if (best == null || total < best.pathCost + + dictionary.connectionCost(best.entry.rightId(), BOUNDARY_CONTEXT)) { + best = node; + } + } + if (best == null) { + throw new IllegalStateException( + "no segmentation path for \"" + text.subSequence(from, to) + "\""); + } + + final List reversed = new ArrayList<>(); + for (Node node = best; node != null; node = node.previous) { + reversed.add(new Morpheme(new Span(node.start, node.end), + text.substring(node.start, node.end), node.entry.features(), node.unknown)); + } + for (int i = reversed.size() - 1; i >= 0; i--) { + morphemes.add(reversed.get(i)); + } + } + + /** Connects a candidate to the cheapest predecessor ending where it starts. */ + private void relax(Node candidate, List predecessors) { + if (predecessors == null) { + candidate.pathCost = candidate.entry.cost() + + dictionary.connectionCost(BOUNDARY_CONTEXT, candidate.entry.leftId()); + return; + } + for (final Node predecessor : predecessors) { + final long total = predecessor.pathCost + + dictionary.connectionCost(predecessor.entry.rightId(), candidate.entry.leftId()) + + candidate.entry.cost(); + if (total < candidate.pathCost) { + candidate.pathCost = total; + candidate.previous = predecessor; + } + } + } + + /** Gathers lexicon matches and unknown-word candidates starting at one position. */ + private List candidates(String text, int from, int to, int offset) { + final int position = from + offset; + final List candidates = new ArrayList<>(); + final int longest = Math.min(dictionary.maxSurfaceLength(), to - position); + boolean lexiconMatch = false; + for (int length = 1; length <= longest; length++) { + final List entries = + dictionary.lookup(text.substring(position, position + length)); + if (entries == null) { + continue; + } + lexiconMatch = true; + for (final WordEntry entry : entries) { + candidates.add(new Node(position, position + length, entry, false)); + } + } + + final Category category = dictionary.categoryOf(text.charAt(position)); + if (!lexiconMatch || category.invoke()) { + int run = position + 1; + while (run < to + && dictionary.categoryOf(text.charAt(run)).name().equals(category.name())) { + run++; + } + final List templates = dictionary.unknownEntries(category.name()); + if (templates != null) { + addUnknown(candidates, position, run, to, category, templates); + } + } + if (candidates.isEmpty()) { + // no entry and no template: a single-character fallback keeps the lattice alive + final List fallback = dictionary.unknownEntries("DEFAULT"); + if (fallback != null) { + for (final WordEntry entry : fallback) { + candidates.add(new Node(position, position + 1, entry, true)); + } + } + } + if (candidates.isEmpty()) { + throw new IllegalStateException("dictionary provides no candidate at position " + + position + "; unk.def lacks a DEFAULT template"); + } + return candidates; + } + + /** Emits unknown-word candidates per the category's grouping and length settings. */ + private static void addUnknown(List candidates, int position, int runEnd, + int to, Category category, List templates) { + if (category.group()) { + for (final WordEntry entry : templates) { + candidates.add(new Node(position, runEnd, entry, true)); + } + } + final int lengths = category.length(); + for (int length = 1; length <= lengths && position + length <= to; length++) { + if (category.group() && position + length == runEnd) { + continue; // already emitted as the grouped run + } + for (final WordEntry entry : templates) { + candidates.add(new Node(position, position + length, entry, true)); + } + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java new file mode 100644 index 0000000000..5e1c60f1a6 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.util.StringUtil; + +/** + * An immutable, in-memory dictionary in the mecab directory format: lexicon entries + * from the {@code *.csv} files, connection costs from {@code matrix.def}, character + * categories from {@code char.def}, and unknown-word templates from {@code unk.def}, + * loaded from a user-supplied dictionary directory. The reader is a clean-room + * implementation of the documented format; no dictionary data is bundled or downloaded + * by this class, so the dictionaries' own licenses never attach to this library. + * + *

The same format serves multiple languages: the Japanese IPADIC and UniDic + * distributions and the Korean mecab-ko-dic all load through this one reader, with the + * feature columns passed through untouched because their schemas differ.

+ * + *

Instances are immutable and safe to share between threads.

+ * + * @see LatticeTokenizer + * @since 3.0.0 + */ +public final class MecabDictionary { + + /** One lexicon or unknown-word entry. */ + record WordEntry(int leftId, int rightId, int cost, List features) { + } + + /** One character category's unknown-word behavior from {@code char.def}. */ + record Category(String name, boolean invoke, boolean group, int length) { + } + + private final Map> lexicon; + private final int maxSurfaceLength; + private final short[] connectionCosts; + private final int rightSize; + private final Map categories; + private final String[] categoryOfChar; + private final Map> unknownEntries; + + private MecabDictionary(Map> lexicon, int maxSurfaceLength, + short[] connectionCosts, int rightSize, Map categories, + String[] categoryOfChar, Map> unknownEntries) { + this.lexicon = lexicon; + this.maxSurfaceLength = maxSurfaceLength; + this.connectionCosts = connectionCosts; + this.rightSize = rightSize; + this.categories = categories; + this.categoryOfChar = categoryOfChar; + this.unknownEntries = unknownEntries; + } + + /** + * Loads a dictionary directory encoded in UTF-8. + * + * @param directory The unpacked dictionary directory. Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails or a file is malformed. + * @throws IllegalArgumentException Thrown if {@code directory} is {@code null}. + */ + public static MecabDictionary load(Path directory) throws IOException { + return load(directory, StandardCharsets.UTF_8); + } + + /** + * Loads a dictionary directory. + * + * @param directory The unpacked dictionary directory holding the {@code *.csv} + * lexicon files, {@code matrix.def}, {@code char.def}, and + * {@code unk.def}. Must not be {@code null}. + * @param charset The encoding the distribution uses, for example UTF-8 or EUC-JP. + * Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails, a required file is missing, or a file + * is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static MecabDictionary load(Path directory, Charset charset) throws IOException { + if (directory == null || charset == null) { + throw new IllegalArgumentException("directory and charset must not be null"); + } + final Map> lexicon = new HashMap<>(); + int maxSurface = 0; + try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { + for (final Path csv : csvFiles) { + maxSurface = Math.max(maxSurface, readLexicon(csv, charset, lexicon)); + } + } + if (lexicon.isEmpty()) { + throw new IOException("no lexicon entries found under " + directory); + } + + final List matrixLines = readLines(directory.resolve("matrix.def"), charset); + if (matrixLines.isEmpty()) { + throw new IOException("empty matrix.def under " + directory); + } + final String[] header = splitWhitespace(matrixLines.get(0)); + if (header.length != 2) { + throw new IOException("malformed matrix.def header: " + matrixLines.get(0)); + } + final int leftSize = parseInt(header[0], "matrix.def", 1); + final int rightSize = parseInt(header[1], "matrix.def", 1); + final short[] costs = new short[leftSize * rightSize]; + for (int i = 1; i < matrixLines.size(); i++) { + final String line = matrixLines.get(i).trim(); + if (line.isEmpty()) { + continue; + } + final String[] fields = splitWhitespace(line); + if (fields.length != 3) { + throw new IOException("malformed matrix.def line " + (i + 1)); + } + final int right = parseInt(fields[0], "matrix.def", i + 1); + final int left = parseInt(fields[1], "matrix.def", i + 1); + costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); + } + + final Map categories = new HashMap<>(); + final String[] categoryOfChar = new String[Character.MAX_VALUE + 1]; + readCharacterDefinition(directory.resolve("char.def"), charset, categories, + categoryOfChar); + + final Map> unknown = new HashMap<>(); + readLexicon(directory.resolve("unk.def"), charset, unknown); + + return new MecabDictionary(lexicon, maxSurface, costs, rightSize, categories, + categoryOfChar, unknown); + } + + /** Reads one lexicon-format CSV file; returns the longest surface seen. */ + private static int readLexicon(Path file, Charset charset, + Map> target) throws IOException { + int maxSurface = 0; + int lineNumber = 0; + for (final String line : readLines(file, charset)) { + lineNumber++; + if (line.isEmpty()) { + continue; + } + final List fields = splitCsv(line); + if (fields.size() < 4) { + throw new IOException("malformed entry at " + file + " line " + lineNumber); + } + final String surface = fields.get(0); + if (surface.isEmpty()) { + continue; + } + final WordEntry entry = new WordEntry( + parseInt(fields.get(1), file.toString(), lineNumber), + parseInt(fields.get(2), file.toString(), lineNumber), + parseInt(fields.get(3), file.toString(), lineNumber), + List.copyOf(fields.subList(4, fields.size()))); + target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); + maxSurface = Math.max(maxSurface, surface.length()); + } + return maxSurface; + } + + /** Reads char.def: category behavior lines and code point mapping lines. */ + private static void readCharacterDefinition(Path file, Charset charset, + Map categories, String[] categoryOfChar) throws IOException { + int lineNumber = 0; + for (final String raw : readLines(file, charset)) { + lineNumber++; + final String line = stripComment(raw).trim(); + if (line.isEmpty()) { + continue; + } + final String[] fields = splitWhitespace(line); + if (fields[0].startsWith("0x") || fields[0].startsWith("0X")) { + final int rangeSeparator = fields[0].indexOf(".."); + final int from; + final int to; + if (rangeSeparator >= 0) { + from = parseCodePoint(fields[0].substring(0, rangeSeparator), file, lineNumber); + to = parseCodePoint(fields[0].substring(rangeSeparator + 2), file, lineNumber); + } else { + from = parseCodePoint(fields[0], file, lineNumber); + to = from; + } + if (fields.length < 2) { + throw new IOException("mapping without category at " + file + " line " + lineNumber); + } + for (int c = from; c <= to && c <= Character.MAX_VALUE; c++) { + categoryOfChar[c] = fields[1]; + } + } else { + if (fields.length < 4) { + throw new IOException("malformed category at " + file + " line " + lineNumber); + } + categories.put(fields[0], new Category(fields[0], + "1".equals(fields[1]), "1".equals(fields[2]), + parseInt(fields[3], file.toString(), lineNumber))); + } + } + if (!categories.containsKey("DEFAULT")) { + throw new IOException("char.def defines no DEFAULT category: " + file); + } + } + + /** + * Looks up the lexicon entries for an exact surface form. + * + * @param surface The surface form. + * @return The entries, or {@code null} when the surface is not listed. + */ + List lookup(String surface) { + return lexicon.get(surface); + } + + /** @return The longest surface form in the lexicon, bounding prefix enumeration. */ + int maxSurfaceLength() { + return maxSurfaceLength; + } + + /** + * Reads the connection cost between two adjacent nodes. + * + * @param rightId The right context id of the earlier node. + * @param leftId The left context id of the later node. + * @return The connection cost. + */ + int connectionCost(int rightId, int leftId) { + return connectionCosts[rightId * rightSize + leftId]; + } + + /** + * Classifies a character. + * + * @param c The character. + * @return Its category, falling back to {@code DEFAULT}. Never {@code null}. + */ + Category categoryOf(char c) { + final String name = categoryOfChar[c]; + final Category category = name == null ? null : categories.get(name); + return category != null ? category : categories.get("DEFAULT"); + } + + /** + * Looks up the unknown-word templates of a category. + * + * @param category The category name. + * @return The templates, or {@code null} when the category has none. + */ + List unknownEntries(String category) { + return unknownEntries.get(category); + } + + private static List readLines(Path file, Charset charset) throws IOException { + if (!Files.exists(file)) { + throw new IOException("required dictionary file is missing: " + file); + } + final String content = new String(Files.readAllBytes(file), charset); + final List lines = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= content.length(); i++) { + if (i == content.length() || content.charAt(i) == '\n') { + int end = i; + if (end > start && content.charAt(end - 1) == '\r') { + end--; + } + lines.add(content.substring(start, end)); + start = i + 1; + } + } + return lines; + } + + private static String stripComment(String line) { + final int hash = line.indexOf('#'); + return hash < 0 ? line : line.substring(0, hash); + } + + private static List splitCsv(String line) { + final List fields = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || line.charAt(i) == ',') { + fields.add(line.substring(start, i)); + start = i + 1; + } + } + return fields; + } + + private static String[] splitWhitespace(String line) { + final List parts = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || StringUtil.isWhitespace(line.charAt(i))) { + if (start >= 0) { + parts.add(line.substring(start, i)); + start = -1; + } + } else if (start < 0) { + start = i; + } + } + return parts.toArray(new String[0]); + } + + private static int parseInt(String text, String file, int lineNumber) + throws IOException { + try { + return Integer.parseInt(text.trim()); + } catch (NumberFormatException e) { + throw new IOException("malformed number in " + file + " line " + lineNumber, e); + } + } + + private static int parseCodePoint(String text, Path file, int lineNumber) + throws IOException { + try { + return Integer.parseInt(text.trim().substring(2), 16); + } catch (RuntimeException e) { + throw new IOException("malformed code point in " + file + " line " + lineNumber, e); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java new file mode 100644 index 0000000000..33f3ef58dd --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.zip.GZIPInputStream; + +/** + * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the + * dictionary is acquired by the user at install time and never ships with this library. + * The user supplies the archive location from the dictionary project of their choice + * and thereby accepts that dictionary's license; nothing is bundled and no location is + * built in. + * + *

The installer reads gzip-compressed tar archives, the format the common + * distributions use, and extracts only the files a {@link MecabDictionary} reads: + * {@code *.csv}, {@code *.def}, and {@code dicrc}. Entries are flattened to their base + * names, which also means no archive path can escape the target directory.

+ * + * @since 3.0.0 + */ +public final class MecabDictionaryInstaller { + + private static final int TAR_BLOCK = 512; + private static final int TAR_NAME_LENGTH = 100; + private static final int TAR_SIZE_OFFSET = 124; + private static final int TAR_SIZE_LENGTH = 12; + private static final int TAR_TYPE_OFFSET = 156; + + private MecabDictionaryInstaller() { + // static installer only + } + + /** + * Downloads a dictionary archive and unpacks it. + * + * @param archive The archive location, a gzip-compressed tar. Must not be + * {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if fetching, reading, or writing fails, or the archive + * contains no dictionary file. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int install(URI archive, Path targetDirectory) throws IOException { + if (archive == null || targetDirectory == null) { + throw new IllegalArgumentException("archive and targetDirectory must not be null"); + } + try (InputStream in = archive.toURL().openStream()) { + return extract(in, targetDirectory); + } + } + + /** + * Unpacks a dictionary archive stream. + * + * @param archiveStream The gzip-compressed tar content. Must not be {@code null}. + * Not closed. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if reading or writing fails, or the archive contains no + * dictionary file. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int extract(InputStream archiveStream, Path targetDirectory) + throws IOException { + if (archiveStream == null || targetDirectory == null) { + throw new IllegalArgumentException("stream and targetDirectory must not be null"); + } + Files.createDirectories(targetDirectory); + final InputStream tar = new GZIPInputStream(archiveStream); + final byte[] header = new byte[TAR_BLOCK]; + int extracted = 0; + while (readBlock(tar, header)) { + if (isEndBlock(header)) { + break; + } + final String name = headerName(header); + final long size = headerSize(header); + final char type = (char) header[TAR_TYPE_OFFSET]; + final String baseName = baseName(name); + final boolean wanted = (type == '0' || type == 0) + && (baseName.endsWith(".csv") || baseName.endsWith(".def") + || "dicrc".equals(baseName)); + if (wanted) { + final Path file = targetDirectory.resolve(baseName); + try (InputStream entry = boundedStream(tar, size)) { + Files.copy(entry, file, StandardCopyOption.REPLACE_EXISTING); + } + extracted++; + skip(tar, padding(size)); + } else { + skip(tar, size + padding(size)); + } + } + if (extracted == 0) { + throw new IOException("the archive contains no dictionary file"); + } + return extracted; + } + + private static boolean readBlock(InputStream in, byte[] block) throws IOException { + int filled = 0; + while (filled < block.length) { + final int read = in.read(block, filled, block.length - filled); + if (read < 0) { + if (filled == 0) { + return false; + } + throw new IOException("truncated tar header"); + } + filled += read; + } + return true; + } + + private static boolean isEndBlock(byte[] block) { + for (final byte b : block) { + if (b != 0) { + return false; + } + } + return true; + } + + private static String headerName(byte[] header) { + int end = 0; + while (end < TAR_NAME_LENGTH && header[end] != 0) { + end++; + } + return new String(header, 0, end, StandardCharsets.UTF_8); + } + + private static long headerSize(byte[] header) throws IOException { + long size = 0; + for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) { + final byte b = header[i]; + if (b == 0 || b == ' ') { + continue; + } + if (b < '0' || b > '7') { + throw new IOException("malformed tar size field"); + } + size = size * 8 + (b - '0'); + } + return size; + } + + private static String baseName(String name) { + final int slash = name.lastIndexOf('/'); + return slash < 0 ? name : name.substring(slash + 1); + } + + private static long padding(long size) { + final long remainder = size % TAR_BLOCK; + return remainder == 0 ? 0 : TAR_BLOCK - remainder; + } + + private static void skip(InputStream in, long bytes) throws IOException { + long remaining = bytes; + final byte[] buffer = new byte[8192]; + while (remaining > 0) { + final int read = in.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + } + } + + /** Wraps the tar stream so exactly one entry's bytes are readable. */ + private static InputStream boundedStream(InputStream in, long size) { + return new InputStream() { + private long remaining = size; + + @Override + public int read() throws IOException { + if (remaining <= 0) { + return -1; + } + final int b = in.read(); + if (b < 0) { + throw new IOException("truncated tar entry"); + } + remaining--; + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + if (remaining <= 0) { + return -1; + } + final int read = in.read(buffer, offset, (int) Math.min(length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + return read; + } + }; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java new file mode 100644 index 0000000000..a6694ddc58 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.util.List; + +import opennlp.tools.util.Span; + +/** + * One morpheme from lattice segmentation: the {@link Span} it covers in the original + * text, its surface form, and the feature columns its dictionary entry carries. + * + *

The features are the entry's columns exactly as listed in the dictionary, since + * different dictionaries carry different schemas: part of speech first by convention, + * then dictionary-specific columns such as conjugation, base form, or reading. A + * morpheme produced by unknown-word handling has the unknown entry's features and is + * marked as such.

+ * + * @param span The location of the morpheme in the original text. Must not be + * {@code null}. + * @param surface The covered text. Must not be {@code null} or empty. + * @param features The dictionary feature columns. Must not be {@code null}. + * @param unknown Whether the morpheme came from unknown-word handling rather than a + * lexicon entry. + * + * @since 3.0.0 + */ +public record Morpheme(Span span, String surface, List features, + boolean unknown) { + + /** + * Validates the morpheme. + * + * @throws IllegalArgumentException Thrown if {@code span}, {@code surface}, or + * {@code features} is {@code null}, or {@code surface} is empty. + */ + public Morpheme { + if (span == null) { + throw new IllegalArgumentException("span must not be null"); + } + if (surface == null || surface.isEmpty()) { + throw new IllegalArgumentException("surface must not be null or empty"); + } + if (features == null) { + throw new IllegalArgumentException("features must not be null"); + } + features = List.copyOf(features); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java new file mode 100644 index 0000000000..1c3f40a8d3 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.Span; + +/** + * Tests the lattice segmenter against a project-authored miniature dictionary; no + * external dictionary data is involved. + */ +public class LatticeTokenizerTest { + + @TempDir + static Path directory; + + private static LatticeTokenizer tokenizer; + + @BeforeAll + static void loadDictionary() throws IOException { + write("lexicon.csv", String.join("\n", + "東京,0,0,3000,noun,proper", + "京都,0,0,3000,noun,proper", + "東,0,0,6000,noun,common", + "都,0,0,4000,noun,suffix", + "に,0,0,1000,particle,case", + "行く,0,0,3000,verb,base", + "")); + write("matrix.def", "1 1\n0 0 0\n"); + write("char.def", String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "LATIN 1 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "0x0041..0x005A LATIN", + "0x0061..0x007A LATIN", + "")); + write("unk.def", String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "LATIN,0,0,4000,noun,foreign", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")); + tokenizer = new LatticeTokenizer(MecabDictionary.load(directory)); + } + + private static void write(String name, String content) throws IOException { + Files.write(directory.resolve(name), content.getBytes(StandardCharsets.UTF_8)); + } + + @Test + void testLatticePrefersTheCheaperSegmentation() { + // the famous case: Tokyo+Metro must win over East+Kyoto + final String text = "東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + } + + @Test + void testMorphemesCarryDictionaryFeatures() { + final List morphemes = + tokenizer.analyze("東京都に行く"); + Assertions.assertEquals(4, morphemes.size()); + Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); + Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); + Assertions.assertEquals(false, morphemes.get(0).unknown()); + } + + @Test + void testUnknownLatinRunGroupsIntoOneMorpheme() { + final List morphemes = tokenizer.analyze("ABCに行く"); + Assertions.assertEquals(3, morphemes.size()); + Assertions.assertEquals("ABC", morphemes.get(0).surface()); + Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(0).features()); + } + + @Test + void testUnknownKanjiPreferOneMorphemeOverTwo() { + final List morphemes = tokenizer.analyze("峠道に行く"); + Assertions.assertEquals(3, morphemes.size()); + Assertions.assertEquals("峠道", morphemes.get(0).surface()); + Assertions.assertEquals(true, morphemes.get(0).unknown()); + } + + @Test + void testWhitespaceSeparatesAndIsNeverAMorpheme() { + final String text = "東京 に 行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(3, 4), new Span(5, 7)}, + tokenizer.tokenizePos(text)); + Assertions.assertEquals(0, tokenizer.analyze(" ").size()); + Assertions.assertEquals(0, tokenizer.analyze("").size()); + } + + @Test + void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + + Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("char.def"), + "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("unk.def"), + "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new LatticeTokenizer(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java new file mode 100644 index 0000000000..fe999e4609 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.GZIPOutputStream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class MecabDictionaryInstallerTest { + + /** Writes one ustar entry: a 512-byte header block and padded content. */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] header = new byte[512]; + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } + + private static byte[] archive(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + @Test + void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) + throws IOException { + final byte[] archive = archive(new String[][] { + {"dict-1.0/lexicon.csv", "cat,0,0,100,noun\n"}, + {"dict-1.0/matrix.def", "1 1\n0 0 0\n"}, + {"dict-1.0/char.def", "DEFAULT 0 1 0\n"}, + {"dict-1.0/unk.def", "DEFAULT,0,0,10000,unknown\n"}, + {"dict-1.0/README", "not a dictionary file"}, + {"dict-1.0/dicrc", "config"}}); + + final int extracted = MecabDictionaryInstaller.extract( + new ByteArrayInputStream(archive), target); + + Assertions.assertEquals(5, extracted); + Assertions.assertTrue(Files.exists(target.resolve("lexicon.csv"))); + Assertions.assertTrue(Files.exists(target.resolve("matrix.def"))); + Assertions.assertTrue(Files.exists(target.resolve("char.def"))); + Assertions.assertTrue(Files.exists(target.resolve("unk.def"))); + Assertions.assertTrue(Files.exists(target.resolve("dicrc"))); + Assertions.assertTrue(Files.notExists(target.resolve("README"))); + Assertions.assertEquals("cat,0,0,100,noun\n", + Files.readString(target.resolve("lexicon.csv"))); + } + + @Test + void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) + throws IOException { + final Path archiveFile = source.resolve("dict.tar.gz"); + Files.write(archiveFile, archive(new String[][] { + {"d/words.csv", "cat,0,0,100,noun\n"}, + {"d/matrix.def", "1 1\n0 0 0\n"}})); + + final int extracted = + MecabDictionaryInstaller.install(archiveFile.toUri(), target); + + Assertions.assertEquals(2, extracted); + Assertions.assertTrue(Files.exists(target.resolve("words.csv"))); + } + + @Test + void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) + throws IOException { + final byte[] archive = archive(new String[][] {{"readme.txt", "nothing here"}}); + Assertions.assertThrows(IOException.class, () -> MecabDictionaryInstaller.extract( + new ByteArrayInputStream(archive), target)); + } + + @Test + void testInvalidArguments(@TempDir Path target) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.install(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.extract(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.extract( + new ByteArrayInputStream(new byte[0]), null)); + } +} From 781c222147578f7f0ba5336e4405314f0ec1a3e7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:56:19 -0400 Subject: [PATCH 02/24] OPENNLP-1894: Character trie for lattice prefix search Common-prefix lookup walks a trie built at load time instead of probing substrings per length, terminating on the first missing prefix and allocating nothing per position. (cherry picked from commit e10ce4b2b8d47821f4ffffcc1da109dcdb7219ba) --- .../tokenize/lattice/LatticeTokenizer.java | 15 ++-- .../tokenize/lattice/MecabDictionary.java | 68 +++++++++++++++++-- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index c022374a5e..c811fd3266 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -203,19 +203,14 @@ private void relax(Node candidate, List predecessors) { private List candidates(String text, int from, int to, int offset) { final int position = from + offset; final List candidates = new ArrayList<>(); - final int longest = Math.min(dictionary.maxSurfaceLength(), to - position); - boolean lexiconMatch = false; - for (int length = 1; length <= longest; length++) { - final List entries = - dictionary.lookup(text.substring(position, position + length)); - if (entries == null) { - continue; - } - lexiconMatch = true; + final boolean[] matched = new boolean[1]; + dictionary.prefixMatches(text, position, to, (length, entries) -> { + matched[0] = true; for (final WordEntry entry : entries) { candidates.add(new Node(position, position + length, entry, false)); } - } + }); + final boolean lexiconMatch = matched[0]; final Category category = dictionary.categoryOf(text.charAt(position)); if (!lexiconMatch || category.invoke()) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 5e1c60f1a6..2e20b8c2d5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -57,7 +57,25 @@ record WordEntry(int leftId, int rightId, int cost, List features) { record Category(String name, boolean invoke, boolean group, int length) { } - private final Map> lexicon; + /** One node of the lexicon trie, keyed by the next surface character. */ + private static final class TrieNode { + private final Map children = new HashMap<>(); + private List entries; + } + + /** Receives one common-prefix match during {@link #prefixMatches}. */ + interface PrefixMatchConsumer { + + /** + * Accepts one match. + * + * @param length The matched surface length in characters. + * @param entries The lexicon entries for that surface. + */ + void accept(int length, List entries); + } + + private final TrieNode lexicon; private final int maxSurfaceLength; private final short[] connectionCosts; private final int rightSize; @@ -65,7 +83,7 @@ record Category(String name, boolean invoke, boolean group, int length) { private final String[] categoryOfChar; private final Map> unknownEntries; - private MecabDictionary(Map> lexicon, int maxSurfaceLength, + private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, short[] connectionCosts, int rightSize, Map categories, String[] categoryOfChar, Map> unknownEntries) { this.lexicon = lexicon; @@ -150,8 +168,22 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown); - return new MecabDictionary(lexicon, maxSurface, costs, rightSize, categories, - categoryOfChar, unknown); + return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, + categories, categoryOfChar, unknown); + } + + /** Folds the surface-keyed lexicon into a character trie for prefix search. */ + private static TrieNode buildTrie(Map> lexicon) { + final TrieNode root = new TrieNode(); + for (final Map.Entry> entry : lexicon.entrySet()) { + TrieNode node = root; + final String surface = entry.getKey(); + for (int i = 0; i < surface.length(); i++) { + node = node.children.computeIfAbsent(surface.charAt(i), key -> new TrieNode()); + } + node.entries = List.copyOf(entry.getValue()); + } + return root; } /** Reads one lexicon-format CSV file; returns the longest surface seen. */ @@ -232,7 +264,33 @@ private static void readCharacterDefinition(Path file, Charset charset, * @return The entries, or {@code null} when the surface is not listed. */ List lookup(String surface) { - return lexicon.get(surface); + TrieNode node = lexicon; + for (int i = 0; i < surface.length() && node != null; i++) { + node = node.children.get(surface.charAt(i)); + } + return node == null ? null : node.entries; + } + + /** + * Reports every lexicon surface starting at a text position, walking the trie once + * with no substring allocation. + * + * @param text The text being segmented. + * @param from The position surfaces must start at. + * @param to The exclusive end of the searchable stretch. + * @param consumer Receives each match. + */ + void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { + TrieNode node = lexicon; + for (int i = from; i < to; i++) { + node = node.children.get(text.charAt(i)); + if (node == null) { + return; + } + if (node.entries != null) { + consumer.accept(i - from + 1, node.entries); + } + } } /** @return The longest surface form in the lexicon, bounding prefix enumeration. */ From f1fada5c70e6bdf4555c1dbaa8de8a31624a538a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:58:23 -0400 Subject: [PATCH 03/24] OPENNLP-1894: Frequency-driven segmentation over user-supplied lexicons A Viterbi search maximizing summed word log-probabilities segments Chinese and similar scripts from a plain word-count lexicon, with unlisted characters falling back to single-character words. The user supplies the lexicon and thereby accepts its license; nothing is bundled. (cherry picked from commit bff3f23d4858562171940d09ba2c2321f5005ab0) --- .../tokenize/lattice/UnigramSegmenter.java | 258 ++++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 108 ++++++++ 2 files changed, 366 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java new file mode 100644 index 0000000000..3d1ca9884e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.util.Span; +import opennlp.tools.util.StringUtil; + +/** + * Frequency-driven segmentation for Chinese and similar scripts: a Viterbi search that + * maximizes the summed log-probability of the words in a user-supplied frequency + * lexicon, with unlisted characters falling back to single-character words. This is the + * unigram model behind common Chinese segmenters; it carries no connection costs, so it + * is lighter than the {@link LatticeTokenizer} and fits lexicons that list only words + * and counts. + * + *

The lexicon format is one entry per line: the word, its count, and optionally a + * tag, separated by whitespace. The user supplies the lexicon file and thereby accepts + * its license; nothing is bundled. Every reported span is in original text + * coordinates.

+ * + *

Instances are immutable and safe to share between threads.

+ * + * @since 3.0.0 + */ +public class UnigramSegmenter implements Tokenizer { + + /** The log-probability charged to a character the lexicon does not know. */ + private final double unknownLogProbability; + + private final WordTrie trie; + + /** A minimal trie over words with their log-probabilities. */ + private static final class WordTrie { + private final Map children = new HashMap<>(); + private double logProbability = Double.NaN; + } + + private UnigramSegmenter(WordTrie trie, double unknownLogProbability) { + this.trie = trie; + this.unknownLogProbability = unknownLogProbability; + } + + /** + * Loads a frequency lexicon encoded in UTF-8. + * + * @param lexicon The lexicon file. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if {@code lexicon} is {@code null}. + */ + public static UnigramSegmenter load(Path lexicon) throws IOException { + return load(lexicon, StandardCharsets.UTF_8); + } + + /** + * Loads a frequency lexicon. + * + * @param lexicon The lexicon file: one word, its count, and an optional tag per + * line. Must not be {@code null}. + * @param charset The lexicon encoding. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOException { + if (lexicon == null || charset == null) { + throw new IllegalArgumentException("lexicon and charset must not be null"); + } + try (InputStream in = Files.newInputStream(lexicon)) { + return load(in, charset); + } + } + + /** + * Loads a frequency lexicon from a stream. + * + * @param lexiconStream The lexicon content. Must not be {@code null}. Not closed. + * @param charset The lexicon encoding. Must not be {@code null}. + * @return The segmenter. Never {@code null}. + * @throws IOException Thrown if reading fails or the lexicon is empty or malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) + throws IOException { + if (lexiconStream == null || charset == null) { + throw new IllegalArgumentException("stream and charset must not be null"); + } + final Map counts = new HashMap<>(); + long total = 0; + final String content = new String(lexiconStream.readAllBytes(), charset); + int lineStart = 0; + int lineNumber = 0; + for (int i = 0; i <= content.length(); i++) { + if (i < content.length() && content.charAt(i) != '\n') { + continue; + } + lineNumber++; + final String line = content.substring(lineStart, i).trim(); + lineStart = i + 1; + if (line.isEmpty()) { + continue; + } + final int wordEnd = whitespaceIndex(line); + if (wordEnd < 0) { + throw new IOException("lexicon line " + lineNumber + " has no count"); + } + final String word = line.substring(0, wordEnd); + int countStart = wordEnd; + while (countStart < line.length() && StringUtil.isWhitespace(line.charAt(countStart))) { + countStart++; + } + int countEnd = countStart; + while (countEnd < line.length() && !StringUtil.isWhitespace(line.charAt(countEnd))) { + countEnd++; + } + final long count; + try { + count = Long.parseLong(line.substring(countStart, countEnd)); + } catch (NumberFormatException e) { + throw new IOException("malformed count at lexicon line " + lineNumber, e); + } + if (count <= 0) { + throw new IOException("count must be positive at lexicon line " + lineNumber); + } + counts.merge(word, count, Long::sum); + total += count; + } + if (counts.isEmpty()) { + throw new IOException("the lexicon lists no words"); + } + + final WordTrie root = new WordTrie(); + final double logTotal = Math.log(total); + for (final Map.Entry entry : counts.entrySet()) { + WordTrie node = root; + final String word = entry.getKey(); + for (int c = 0; c < word.length(); c++) { + node = node.children.computeIfAbsent(word.charAt(c), + key -> new WordTrie()); + } + node.logProbability = Math.log(entry.getValue()) - logTotal; + } + // rarer than any listed word: a fraction of a single count + final double unknown = Math.log(0.5) - logTotal; + return new UnigramSegmenter(root, unknown); + } + + @Override + public String[] tokenize(String s) { + final Span[] spans = tokenizePos(s); + final String[] tokens = new String[spans.length]; + for (int i = 0; i < tokens.length; i++) { + tokens[i] = s.substring(spans[i].getStart(), spans[i].getEnd()); + } + return tokens; + } + + @Override + public Span[] tokenizePos(String s) { + if (s == null) { + throw new IllegalArgumentException("text must not be null"); + } + final List spans = new ArrayList<>(); + int start = 0; + while (start < s.length()) { + if (StringUtil.isWhitespace(s.charAt(start))) { + start++; + continue; + } + int end = start; + while (end < s.length() && !StringUtil.isWhitespace(s.charAt(end))) { + end++; + } + decode(s, start, end, spans); + start = end; + } + return spans.toArray(new Span[0]); + } + + /** Viterbi over word log-probabilities within one whitespace-free stretch. */ + private void decode(String text, int from, int to, List spans) { + final int length = to - from; + final double[] best = new double[length + 1]; + final int[] previous = new int[length + 1]; + for (int i = 1; i <= length; i++) { + best[i] = Double.NEGATIVE_INFINITY; + } + for (int i = 0; i < length; i++) { + if (best[i] == Double.NEGATIVE_INFINITY) { + continue; + } + // the single-character fallback keeps every position reachable + final double fallback = best[i] + unknownLogProbability; + if (fallback > best[i + 1]) { + best[i + 1] = fallback; + previous[i + 1] = i; + } + WordTrie node = trie; + for (int j = from + i; j < to; j++) { + node = node.children.get(text.charAt(j)); + if (node == null) { + break; + } + if (!Double.isNaN(node.logProbability)) { + final int end = j - from + 1; + final double score = best[i] + node.logProbability; + if (score > best[end]) { + best[end] = score; + previous[end] = i; + } + } + } + } + final List reversed = new ArrayList<>(); + for (int end = length; end > 0; end = previous[end]) { + reversed.add(new Span(from + previous[end], from + end)); + } + for (int i = reversed.size() - 1; i >= 0; i--) { + spans.add(reversed.get(i)); + } + } + + private static int whitespaceIndex(String text) { + for (int i = 0; i < text.length(); i++) { + if (StringUtil.isWhitespace(text.charAt(i))) { + return i; + } + } + return -1; + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java new file mode 100644 index 0000000000..a6104c1958 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +/** + * Tests the frequency-driven segmenter against a project-authored miniature lexicon; + * no external lexicon data is involved. + */ +public class UnigramSegmenterTest { + + private static final String LEXICON = String.join("\n", + "我 5000 r", + "来到 2000 v", + "北京 3000 ns", + "清华大学 800 nt", + "清华 400 ns", + "华大 100 ns", + "大学 1500 n", + "的 9000 uj", + ""); + + private static UnigramSegmenter segmenter; + + @BeforeAll + static void loadLexicon() throws IOException { + segmenter = UnigramSegmenter.load( + new ByteArrayInputStream(LEXICON.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + } + + @Test + void testPrefersWholeWordsOverFragments() { + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京", "清华大学"}, + segmenter.tokenize("我来到北京清华大学")); + } + + @Test + void testSpansStayInOriginalCoordinates() { + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 9)}, + segmenter.tokenizePos("我来到北京清华大学")); + } + + @Test + void testUnknownCharactersFallBackToSingles() { + Assertions.assertArrayEquals( + new String[] {"我", "爱", "北京"}, + segmenter.tokenize("我爱北京")); + } + + @Test + void testWhitespaceSeparates() { + Assertions.assertArrayEquals( + new String[] {"北京", "大学"}, + segmenter.tokenize("北京 大学")); + Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); + } + + @Test + void testMalformedLexiconsFailLoud() { + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("word\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("word abc\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("word 0\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream("\n\n".getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((java.nio.file.Path) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenizePos(null)); + } +} From 05737234a95e5b45046b146899f5ac3a6f9f7a2b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:20:06 -0400 Subject: [PATCH 04/24] OPENNLP-1894: Usage example and edge-case tests for the lattice and unigram segmenters, corrected javadoc --- .../tokenize/lattice/LatticeTokenizer.java | 6 +- .../tokenize/lattice/MecabDictionary.java | 48 ++++- .../lattice/MecabDictionaryInstaller.java | 57 ++++- .../tokenize/lattice/UnigramSegmenter.java | 13 +- .../lattice/LatticeTokenizerTest.java | 149 +++++++++++++ .../lattice/LatticeUsageExampleTest.java | 198 ++++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 67 ++++++ 7 files changed, 529 insertions(+), 9 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index c811fd3266..cb610f74fe 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -225,7 +225,8 @@ private List candidates(String text, int from, int to, int offset) { } } if (candidates.isEmpty()) { - // no entry and no template: a single-character fallback keeps the lattice alive + // Neither the lexicon nor the character's category produced a candidate here, so a + // single-character entry from the DEFAULT template keeps the lattice connected. final List fallback = dictionary.unknownEntries("DEFAULT"); if (fallback != null) { for (final WordEntry entry : fallback) { @@ -251,7 +252,8 @@ private static void addUnknown(List candidates, int position, int runEnd, final int lengths = category.length(); for (int length = 1; length <= lengths && position + length <= to; length++) { if (category.group() && position + length == runEnd) { - continue; // already emitted as the grouped run + // This length coincides with the grouped run emitted above; skip the duplicate. + continue; } for (final WordEntry entry : templates) { candidates.add(new Node(position, position + length, entry, true)); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 2e20b8c2d5..908bc75f2c 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -293,7 +293,7 @@ void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) } } - /** @return The longest surface form in the lexicon, bounding prefix enumeration. */ + /** @return The length in characters of the longest surface form in the lexicon. */ int maxSurfaceLength() { return maxSurfaceLength; } @@ -331,6 +331,14 @@ List unknownEntries(String category) { return unknownEntries.get(category); } + /** + * Reads a required dictionary file into its lines, accepting LF and CRLF endings. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @return The lines without their line terminators. Never {@code null}. + * @throws IOException Thrown if the file is missing or reading fails. + */ private static List readLines(Path file, Charset charset) throws IOException { if (!Files.exists(file)) { throw new IOException("required dictionary file is missing: " + file); @@ -351,11 +359,25 @@ private static List readLines(Path file, Charset charset) throws IOExcep return lines; } + /** + * Removes a trailing {@code #} comment from a {@code char.def} line. + * + * @param line The raw line. + * @return The line up to but excluding the first {@code #}, or the whole line when + * there is none. + */ private static String stripComment(String line) { final int hash = line.indexOf('#'); return hash < 0 ? line : line.substring(0, hash); } + /** + * Splits a lexicon line at every comma. Surfaces containing commas are not + * representable, matching the plain-text lexicon format. + * + * @param line The line to split. + * @return The fields in order, empty fields included. Never {@code null}. + */ private static List splitCsv(String line) { final List fields = new ArrayList<>(); int start = 0; @@ -368,6 +390,12 @@ private static List splitCsv(String line) { return fields; } + /** + * Splits a line into its whitespace-separated fields. + * + * @param line The line to split. + * @return The non-empty fields in order. Never {@code null}. + */ private static String[] splitWhitespace(String line) { final List parts = new ArrayList<>(); int start = -1; @@ -384,6 +412,15 @@ private static String[] splitWhitespace(String line) { return parts.toArray(new String[0]); } + /** + * Parses a decimal integer field, reporting the file and line on failure. + * + * @param text The field text. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed value. + * @throws IOException Thrown if the field is not a valid integer. + */ private static int parseInt(String text, String file, int lineNumber) throws IOException { try { @@ -393,6 +430,15 @@ private static int parseInt(String text, String file, int lineNumber) } } + /** + * Parses a {@code 0x}-prefixed hexadecimal code point from {@code char.def}. + * + * @param text The field text including the {@code 0x} prefix. + * @param file The file being read, for the error message. + * @param lineNumber The line being read, for the error message. + * @return The parsed code point. + * @throws IOException Thrown if the field is not a valid hexadecimal code point. + */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { try { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 33f3ef58dd..9bbbde7902 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -34,9 +34,11 @@ * built in. * *

The installer reads gzip-compressed tar archives, the format the common - * distributions use, and extracts only the files a {@link MecabDictionary} reads: - * {@code *.csv}, {@code *.def}, and {@code dicrc}. Entries are flattened to their base - * names, which also means no archive path can escape the target directory.

+ * distributions use, and extracts only the dictionary payload: the {@code *.csv} + * lexicon files and {@code *.def} definition files that a {@link MecabDictionary} + * reads, plus the {@code dicrc} configuration file distributions ship alongside them. + * Entries are flattened to their base names, which also means no archive path can + * escape the target directory.

* * @since 3.0.0 */ @@ -49,7 +51,7 @@ public final class MecabDictionaryInstaller { private static final int TAR_TYPE_OFFSET = 156; private MecabDictionaryInstaller() { - // static installer only + // This class exposes only static methods and is never instantiated. } /** @@ -122,6 +124,15 @@ public static int extract(InputStream archiveStream, Path targetDirectory) return extracted; } + /** + * Fills one tar block from the stream. + * + * @param in The tar stream. + * @param block The block buffer to fill completely. + * @return {@code true} when a full block was read, {@code false} at a clean end of + * stream before any byte of the block. + * @throws IOException Thrown if the stream ends inside the block or reading fails. + */ private static boolean readBlock(InputStream in, byte[] block) throws IOException { int filled = 0; while (filled < block.length) { @@ -137,6 +148,12 @@ private static boolean readBlock(InputStream in, byte[] block) throws IOExceptio return true; } + /** + * Recognizes the all-zero block that terminates a tar archive. + * + * @param block The block to inspect. + * @return {@code true} when every byte is zero. + */ private static boolean isEndBlock(byte[] block) { for (final byte b : block) { if (b != 0) { @@ -146,6 +163,12 @@ private static boolean isEndBlock(byte[] block) { return true; } + /** + * Reads the NUL-terminated entry name from a tar header block. + * + * @param header The header block. + * @return The entry name. Never {@code null}. + */ private static String headerName(byte[] header) { int end = 0; while (end < TAR_NAME_LENGTH && header[end] != 0) { @@ -154,6 +177,13 @@ private static String headerName(byte[] header) { return new String(header, 0, end, StandardCharsets.UTF_8); } + /** + * Reads the octal entry size from a tar header block. + * + * @param header The header block. + * @return The entry size in bytes. + * @throws IOException Thrown if the size field holds a non-octal digit. + */ private static long headerSize(byte[] header) throws IOException { long size = 0; for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) { @@ -169,16 +199,35 @@ private static long headerSize(byte[] header) throws IOException { return size; } + /** + * Strips any directory prefix from an archive entry name. + * + * @param name The entry name as stored in the archive. + * @return The part after the last {@code /}, or the whole name when there is none. + */ private static String baseName(String name) { final int slash = name.lastIndexOf('/'); return slash < 0 ? name : name.substring(slash + 1); } + /** + * Computes the padding after an entry: tar content is stored in whole blocks. + * + * @param size The entry size in bytes. + * @return The number of padding bytes up to the next block boundary. + */ private static long padding(long size) { final long remainder = size % TAR_BLOCK; return remainder == 0 ? 0 : TAR_BLOCK - remainder; } + /** + * Consumes and discards an exact number of bytes from the stream. + * + * @param in The stream to read from. + * @param bytes The number of bytes to discard. + * @throws IOException Thrown if the stream ends before that many bytes were read. + */ private static void skip(InputStream in, long bytes) throws IOException { long remaining = bytes; final byte[] buffer = new byte[8192]; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index 3d1ca9884e..ea3d188bc5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -167,7 +167,8 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) } node.logProbability = Math.log(entry.getValue()) - logTotal; } - // rarer than any listed word: a fraction of a single count + // Charge an unlisted character half of one count out of the total, which makes it + // rarer than any listed word: every listed count is at least one. final double unknown = Math.log(0.5) - logTotal; return new UnigramSegmenter(root, unknown); } @@ -216,7 +217,8 @@ private void decode(String text, int from, int to, List spans) { if (best[i] == Double.NEGATIVE_INFINITY) { continue; } - // the single-character fallback keeps every position reachable + // A single-character step at the unknown log-probability keeps every position + // reachable even where no lexicon word matches. final double fallback = best[i] + unknownLogProbability; if (fallback > best[i + 1]) { best[i + 1] = fallback; @@ -247,6 +249,13 @@ private void decode(String text, int from, int to, List spans) { } } + /** + * Finds the first whitespace character in a lexicon line. + * + * @param text The line to scan. + * @return The index of the first whitespace character, or {@code -1} when the line + * contains none. + */ private static int whitespaceIndex(String text) { for (int i = 0; i < text.length(); i++) { if (StringUtil.isWhitespace(text.charAt(i))) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 1c3f40a8d3..b643786a97 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -128,6 +128,155 @@ void testWhitespaceSeparatesAndIsNeverAMorpheme() { Assertions.assertEquals(0, tokenizer.analyze("").size()); } + /** + * Verifies that empty input yields empty results from every view of the tokenizer. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], tokenizer.tokenize("")); + Assertions.assertArrayEquals(new Span[0], tokenizer.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed surface and for an unlisted kanji: + * both come back as exactly one morpheme covering {@code [0, 1)}, and only the + * unlisted one is marked unknown. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"に"}, tokenizer.tokenize("に")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("に")); + Assertions.assertFalse(tokenizer.analyze("に").get(0).unknown()); + + final List unknown = tokenizer.analyze("峠"); + Assertions.assertEquals(1, unknown.size()); + Assertions.assertEquals("峠", unknown.get(0).surface()); + Assertions.assertEquals(new Span(0, 1), unknown.get(0).span()); + Assertions.assertTrue(unknown.get(0).unknown()); + } + + /** + * Verifies input made entirely of characters absent from both the lexicon and the + * {@code char.def} mappings: they fall into the DEFAULT category, whose grouping + * setting joins the whole same-category run into one unknown morpheme carrying the + * DEFAULT template's features. + */ + @Test + void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { + final List morphemes = tokenizer.analyze("①②③"); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals("①②③", morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span()); + Assertions.assertTrue(morphemes.get(0).unknown()); + Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features()); + } + + /** + * Verifies a mixed run of known and unknown text: the lexicon words around an + * unmapped character are kept intact, the unmapped character becomes its own + * unknown morpheme, and every span stays in original text coordinates. + */ + @Test + void testMixedKnownAndUnknownRuns() { + final String text = "東京①に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "①", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + final List morphemes = tokenizer.analyze(text); + Assertions.assertFalse(morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(1).unknown()); + Assertions.assertFalse(morphemes.get(2).unknown()); + } + + /** + * Verifies that spans keep original text coordinates when the interesting content + * does not start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " 東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)}, + tokenizer.tokenizePos(text)); + } + + /** + * Verifies that a lexicon row with fewer than the four mandatory columns is + * rejected at load time. + */ + @Test + void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a non-numeric cost column in a lexicon row is rejected at load + * time. + */ + @Test + void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code matrix.def} data line with the wrong number of fields is + * rejected at load time. + */ + @Test + void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies that a {@code char.def} code point mapping without a category name is + * rejected at load time. + */ + @Test + void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) + throws IOException { + Files.write(broken.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(broken.resolve("char.def"), + "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); + Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + } + + /** + * Verifies the fail-loud path when a loadable dictionary cannot cover the input: the + * {@code unk.def} has no DEFAULT template, so a character with neither a lexicon + * entry nor a category template stops segmentation with an exception instead of + * being dropped silently. + */ + @Test + void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) + throws IOException { + Files.write(partial.resolve("lexicon.csv"), + "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("char.def"), + "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" + .getBytes(StandardCharsets.UTF_8)); + Files.write(partial.resolve("unk.def"), + "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + final LatticeTokenizer limited = + new LatticeTokenizer(MecabDictionary.load(partial)); + Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("①")); + } + @Test void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java new file mode 100644 index 0000000000..a5e2a39d1a --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.zip.GZIPOutputStream; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.Span; + +/** + * Demonstrates the intended end-to-end usage of this package with miniature, + * project-authored data: a mecab-format dictionary archive is installed with + * {@link MecabDictionaryInstaller}, loaded as a {@link MecabDictionary}, and segmented + * with a {@link LatticeTokenizer}; a plain frequency lexicon is loaded and segmented + * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by + * the test itself; no external dictionary or lexicon data and no network access are + * involved. + */ +public class LatticeUsageExampleTest { + + /** + * Appends one ustar file entry to a growing tar image: a 512-byte header block + * followed by the content padded to a block boundary. + * + * @param tar The tar image under construction. Must not be {@code null}. + * @param name The entry name including any directory prefix. Must not be + * {@code null}, empty, or longer than the 100-byte header name field. + * @param content The entry content bytes. Must not be {@code null}. + * @throws IOException Thrown if writing to the in-memory stream fails. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + if (nameBytes.length == 0 || nameBytes.length > 100) { + throw new IllegalArgumentException( + "entry name must be 1..100 bytes, got " + nameBytes.length); + } + final byte[] header = new byte[512]; + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } + + /** + * Builds a gzip-compressed tar archive from name and content pairs, the layout a + * dictionary distribution ships in. + * + * @param entries The entries as {@code {name, content}} pairs. Must not be + * {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + private static byte[] gzippedTar(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + /** + * Walks the full mecab-format flow: package a miniature Japanese dictionary as a + * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The + * segmentation must pick the cheaper path (Tokyo plus the metropolis suffix) over + * the competing reading (east plus Kyoto), the spans must be in original text + * coordinates, and the morphemes must carry the dictionary's feature columns. + */ + @Test + void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) + throws IOException { + // A minimal but complete dictionary: one lexicon file plus the three definition + // files every mecab-format distribution contains, wrapped like a release archive. + final byte[] archive = gzippedTar(new String[][] { + {"mini-dict-0.1/lexicon.csv", String.join("\n", + "東京,0,0,3000,noun,proper", + "京都,0,0,3000,noun,proper", + "東,0,0,6000,noun,common", + "都,0,0,4000,noun,suffix", + "に,0,0,1000,particle,case", + "行く,0,0,3000,verb,base", + "")}, + {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, + {"mini-dict-0.1/char.def", String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "")}, + {"mini-dict-0.1/unk.def", String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")}, + {"mini-dict-0.1/README", "not a dictionary payload file"}}); + final Path archiveFile = work.resolve("mini-dict-0.1.tar.gz"); + Files.write(archiveFile, archive); + + // Install: fetch the archive from the user-chosen location and unpack the payload. + final Path dictionaryDirectory = work.resolve("dictionary"); + final int extracted = + MecabDictionaryInstaller.install(archiveFile.toUri(), dictionaryDirectory); + Assertions.assertEquals(4, extracted); + + // Load and tokenize; both views must agree and stay in original coordinates. + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); + final String text = "東京都に行く"; + Assertions.assertArrayEquals( + new String[] {"東京", "都", "に", "行く"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + + // The analyze view adds the dictionary's feature columns to every morpheme. + final List morphemes = tokenizer.analyze(text); + Assertions.assertEquals(4, morphemes.size()); + Assertions.assertEquals("東京", morphemes.get(0).surface()); + Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); + Assertions.assertFalse(morphemes.get(0).unknown()); + } + + /** + * Walks the frequency-lexicon flow: write a miniature word-count lexicon to a file, + * load it, and segment. The segmentation must recover the listed multi-character + * words with spans in original text coordinates. + */ + @Test + void testLoadAndSegmentWithAFrequencyLexicon(@TempDir Path work) throws IOException { + // One word, its count, and an optional tag per line, whitespace separated. + final Path lexicon = work.resolve("words.txt"); + Files.write(lexicon, String.join("\n", + "我 5000 r", + "来到 2000 v", + "北京 3000 ns", + "天安门 1200 ns", + "").getBytes(StandardCharsets.UTF_8)); + + final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon); + final String text = "我来到北京天安门"; + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京", "天安门"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)}, + segmenter.tokenizePos(text)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index a6104c1958..f086eafa26 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -82,6 +82,73 @@ void testWhitespaceSeparates() { Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); } + /** + * Verifies that empty input yields empty results from both views of the segmenter. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], segmenter.tokenize("")); + Assertions.assertArrayEquals(new Span[0], segmenter.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed word and for a character the + * lexicon does not know: both come back as exactly one token covering + * {@code [0, 1)}. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"我"}, segmenter.tokenize("我")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("我")); + Assertions.assertArrayEquals(new String[] {"爱"}, segmenter.tokenize("爱")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("爱")); + } + + /** + * Verifies input made entirely of characters absent from the lexicon: every + * character becomes its own single-character token, since only the unknown + * fallback is available. + */ + @Test + void testEntirelyUnknownInputFallsBackToSingleCharacters() { + Assertions.assertArrayEquals( + new String[] {"x", "y", "z"}, + segmenter.tokenize("xyz")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 3)}, + segmenter.tokenizePos("xyz")); + } + + /** + * Verifies a mixed run of known and unknown text inside one whitespace-free + * stretch: the unknown character becomes a single token while the listed words + * around it, including the longest listed compound, stay intact. + */ + @Test + void testMixedKnownAndUnknownRuns() { + Assertions.assertArrayEquals( + new String[] {"我", "爱", "清华大学"}, + segmenter.tokenize("我爱清华大学")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 6)}, + segmenter.tokenizePos("我爱清华大学")); + } + + /** + * Verifies that spans keep original text coordinates when the content does not + * start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " 我来到北京"; + Assertions.assertArrayEquals( + new String[] {"我", "来到", "北京"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 3), new Span(3, 5), new Span(5, 7)}, + segmenter.tokenizePos(text)); + } + @Test void testMalformedLexiconsFailLoud() { Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( From c7fadc913d7063705ded3612f3ae8b531349946a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:32:44 -0400 Subject: [PATCH 05/24] OPENNLP-1894: Keep lattice test sources ASCII-only via Unicode escapes, add an EUC-JP loading example --- .../lattice/LatticeTokenizerTest.java | 68 ++++++++-------- .../lattice/LatticeUsageExampleTest.java | 77 +++++++++++++++---- .../lattice/UnigramSegmenterTest.java | 52 +++++++------ 3 files changed, 126 insertions(+), 71 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index b643786a97..83b823706c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -33,6 +33,10 @@ /** * Tests the lattice segmenter against a project-authored miniature dictionary; no * external dictionary data is involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Japanese dictionary as the sibling usage + * example, whose javadoc spells out each fixture word.

*/ public class LatticeTokenizerTest { @@ -44,12 +48,12 @@ public class LatticeTokenizerTest { @BeforeAll static void loadDictionary() throws IOException { write("lexicon.csv", String.join("\n", - "東京,0,0,3000,noun,proper", - "京都,0,0,3000,noun,proper", - "東,0,0,6000,noun,common", - "都,0,0,4000,noun,suffix", - "に,0,0,1000,particle,case", - "行く,0,0,3000,verb,base", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", "")); write("matrix.def", "1 1\n0 0 0\n"); write("char.def", String.join("\n", @@ -79,9 +83,9 @@ private static void write(String name, String content) throws IOException { @Test void testLatticePrefersTheCheaperSegmentation() { // the famous case: Tokyo+Metro must win over East+Kyoto - final String text = "東京都に行く"; + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -91,7 +95,7 @@ void testLatticePrefersTheCheaperSegmentation() { @Test void testMorphemesCarryDictionaryFeatures() { final List morphemes = - tokenizer.analyze("東京都に行く"); + tokenizer.analyze("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); Assertions.assertEquals(4, morphemes.size()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); @@ -100,7 +104,7 @@ void testMorphemesCarryDictionaryFeatures() { @Test void testUnknownLatinRunGroupsIntoOneMorpheme() { - final List morphemes = tokenizer.analyze("ABCに行く"); + final List morphemes = tokenizer.analyze("ABC\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("ABC", morphemes.get(0).surface()); Assertions.assertEquals(true, morphemes.get(0).unknown()); @@ -109,17 +113,17 @@ void testUnknownLatinRunGroupsIntoOneMorpheme() { @Test void testUnknownKanjiPreferOneMorphemeOverTwo() { - final List morphemes = tokenizer.analyze("峠道に行く"); + final List morphemes = tokenizer.analyze("\u5CE0\u9053\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); - Assertions.assertEquals("峠道", morphemes.get(0).surface()); + Assertions.assertEquals("\u5CE0\u9053", morphemes.get(0).surface()); Assertions.assertEquals(true, morphemes.get(0).unknown()); } @Test void testWhitespaceSeparatesAndIsNeverAMorpheme() { - final String text = "東京 に 行く"; + final String text = "\u6771\u4EAC \u306B \u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(3, 4), new Span(5, 7)}, @@ -144,13 +148,13 @@ void testEmptyInputYieldsEmptyResults() { */ @Test void testSingleCharacterInput() { - Assertions.assertArrayEquals(new String[] {"に"}, tokenizer.tokenize("に")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("に")); - Assertions.assertFalse(tokenizer.analyze("に").get(0).unknown()); + Assertions.assertArrayEquals(new String[] {"\u306B"}, tokenizer.tokenize("\u306B")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, tokenizer.tokenizePos("\u306B")); + Assertions.assertFalse(tokenizer.analyze("\u306B").get(0).unknown()); - final List unknown = tokenizer.analyze("峠"); + final List unknown = tokenizer.analyze("\u5CE0"); Assertions.assertEquals(1, unknown.size()); - Assertions.assertEquals("峠", unknown.get(0).surface()); + Assertions.assertEquals("\u5CE0", unknown.get(0).surface()); Assertions.assertEquals(new Span(0, 1), unknown.get(0).span()); Assertions.assertTrue(unknown.get(0).unknown()); } @@ -163,9 +167,9 @@ void testSingleCharacterInput() { */ @Test void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { - final List morphemes = tokenizer.analyze("①②③"); + final List morphemes = tokenizer.analyze("\u2460\u2461\u2462"); Assertions.assertEquals(1, morphemes.size()); - Assertions.assertEquals("①②③", morphemes.get(0).surface()); + Assertions.assertEquals("\u2460\u2461\u2462", morphemes.get(0).surface()); Assertions.assertEquals(new Span(0, 3), morphemes.get(0).span()); Assertions.assertTrue(morphemes.get(0).unknown()); Assertions.assertEquals(List.of("symbol", "unknown"), morphemes.get(0).features()); @@ -178,9 +182,9 @@ void testEntirelyUnknownInputGroupsIntoOneDefaultMorpheme() { */ @Test void testMixedKnownAndUnknownRuns() { - final String text = "東京①に行く"; + final String text = "\u6771\u4EAC\u2460\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "①", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u2460", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -197,9 +201,9 @@ void testMixedKnownAndUnknownRuns() { */ @Test void testSpansStayOriginalAfterLeadingWhitespace() { - final String text = " 東京都に行く"; + final String text = " \u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(2, 4), new Span(4, 5), new Span(5, 6), new Span(6, 8)}, @@ -213,7 +217,7 @@ void testSpansStayOriginalAfterLeadingWhitespace() { @Test void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -224,7 +228,7 @@ void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { @Test void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -235,7 +239,7 @@ void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException @Test void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -248,7 +252,7 @@ void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); Files.write(broken.resolve("char.def"), "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); @@ -265,7 +269,7 @@ void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) throws IOException { Files.write(partial.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); Files.write(partial.resolve("char.def"), "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" @@ -274,13 +278,13 @@ void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); final LatticeTokenizer limited = new LatticeTokenizer(MecabDictionary.load(partial)); - Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("①")); + Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("\u2460")); } @Test void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { Files.write(broken.resolve("lexicon.csv"), - "東,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index a5e2a39d1a..a61969c902 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -39,6 +40,13 @@ * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by * the test itself; no external dictionary or lexicon data and no network access are * involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only. The + * Japanese fixture words are Tokyo (U+6771 U+4EAC), Kyoto (U+4EAC U+90FD), east + * (U+6771), the metropolis suffix (U+90FD), the case particle ni (U+306B), and the + * verb iku, to go (U+884C U+304F); the Chinese fixture words are wo, I (U+6211), + * laidao, arrive (U+6765 U+5230), Beijing (U+5317 U+4EAC), and Tiananmen + * (U+5929 U+5B89 U+95E8).

*/ public class LatticeUsageExampleTest { @@ -120,12 +128,12 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) // files every mecab-format distribution contains, wrapped like a release archive. final byte[] archive = gzippedTar(new String[][] { {"mini-dict-0.1/lexicon.csv", String.join("\n", - "東京,0,0,3000,noun,proper", - "京都,0,0,3000,noun,proper", - "東,0,0,6000,noun,common", - "都,0,0,4000,noun,suffix", - "に,0,0,1000,particle,case", - "行く,0,0,3000,verb,base", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", "")}, {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, {"mini-dict-0.1/char.def", String.join("\n", @@ -154,9 +162,9 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) // Load and tokenize; both views must agree and stay in original coordinates. final LatticeTokenizer tokenizer = new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); - final String text = "東京都に行く"; + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( - new String[] {"東京", "都", "に", "行く"}, + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, tokenizer.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, @@ -165,7 +173,7 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) // The analyze view adds the dictionary's feature columns to every morpheme. final List morphemes = tokenizer.analyze(text); Assertions.assertEquals(4, morphemes.size()); - Assertions.assertEquals("東京", morphemes.get(0).surface()); + Assertions.assertEquals("\u6771\u4EAC", morphemes.get(0).surface()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertFalse(morphemes.get(0).unknown()); } @@ -180,19 +188,58 @@ void testLoadAndSegmentWithAFrequencyLexicon(@TempDir Path work) throws IOExcept // One word, its count, and an optional tag per line, whitespace separated. final Path lexicon = work.resolve("words.txt"); Files.write(lexicon, String.join("\n", - "我 5000 r", - "来到 2000 v", - "北京 3000 ns", - "天安门 1200 ns", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u5929\u5B89\u95E8 1200 ns", "").getBytes(StandardCharsets.UTF_8)); final UnigramSegmenter segmenter = UnigramSegmenter.load(lexicon); - final String text = "我来到北京天安门"; + final String text = "\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"; Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京", "天安门"}, + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u5929\u5B89\u95E8"}, segmenter.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 8)}, segmenter.tokenizePos(text)); } + + /** + * Walks the non-UTF-8 flow that widely used Japanese distributions require: the same + * miniature dictionary is written to disk encoded in EUC-JP and loaded through the + * charset-taking overload. The segmentation must match the UTF-8 run exactly, which + * shows the encoding is a property of loading, not of tokenization. + */ + @Test + void testLoadAnEucJpEncodedDictionary(@TempDir Path work) throws IOException { + final Charset eucJp = Charset.forName("EUC-JP"); + Files.write(work.resolve("lexicon.csv"), String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "").getBytes(eucJp)); + Files.write(work.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(eucJp)); + Files.write(work.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "").getBytes(eucJp)); + Files.write(work.resolve("unk.def"), String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "").getBytes(eucJp)); + + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(work, eucJp)); + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F")); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index f086eafa26..9725b8c729 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -30,18 +30,22 @@ /** * Tests the frequency-driven segmenter against a project-authored miniature lexicon; * no external lexicon data is involved. + * + *

Source strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Chinese frequency lexicon as the sibling usage + * example, whose javadoc spells out each fixture word.

*/ public class UnigramSegmenterTest { private static final String LEXICON = String.join("\n", - "我 5000 r", - "来到 2000 v", - "北京 3000 ns", - "清华大学 800 nt", - "清华 400 ns", - "华大 100 ns", - "大学 1500 n", - "的 9000 uj", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u6E05\u534E\u5927\u5B66 800 nt", + "\u6E05\u534E 400 ns", + "\u534E\u5927 100 ns", + "\u5927\u5B66 1500 n", + "\u7684 9000 uj", ""); private static UnigramSegmenter segmenter; @@ -56,29 +60,29 @@ static void loadLexicon() throws IOException { @Test void testPrefersWholeWordsOverFragments() { Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京", "清华大学"}, - segmenter.tokenize("我来到北京清华大学")); + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); } @Test void testSpansStayInOriginalCoordinates() { Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 9)}, - segmenter.tokenizePos("我来到北京清华大学")); + segmenter.tokenizePos("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); } @Test void testUnknownCharactersFallBackToSingles() { Assertions.assertArrayEquals( - new String[] {"我", "爱", "北京"}, - segmenter.tokenize("我爱北京")); + new String[] {"\u6211", "\u7231", "\u5317\u4EAC"}, + segmenter.tokenize("\u6211\u7231\u5317\u4EAC")); } @Test void testWhitespaceSeparates() { Assertions.assertArrayEquals( - new String[] {"北京", "大学"}, - segmenter.tokenize("北京 大学")); + new String[] {"\u5317\u4EAC", "\u5927\u5B66"}, + segmenter.tokenize("\u5317\u4EAC \u5927\u5B66")); Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); } @@ -98,10 +102,10 @@ void testEmptyInputYieldsEmptyResults() { */ @Test void testSingleCharacterInput() { - Assertions.assertArrayEquals(new String[] {"我"}, segmenter.tokenize("我")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("我")); - Assertions.assertArrayEquals(new String[] {"爱"}, segmenter.tokenize("爱")); - Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("爱")); + Assertions.assertArrayEquals(new String[] {"\u6211"}, segmenter.tokenize("\u6211")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u6211")); + Assertions.assertArrayEquals(new String[] {"\u7231"}, segmenter.tokenize("\u7231")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u7231")); } /** @@ -127,11 +131,11 @@ void testEntirelyUnknownInputFallsBackToSingleCharacters() { @Test void testMixedKnownAndUnknownRuns() { Assertions.assertArrayEquals( - new String[] {"我", "爱", "清华大学"}, - segmenter.tokenize("我爱清华大学")); + new String[] {"\u6211", "\u7231", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u7231\u6E05\u534E\u5927\u5B66")); Assertions.assertArrayEquals(new Span[] { new Span(0, 1), new Span(1, 2), new Span(2, 6)}, - segmenter.tokenizePos("我爱清华大学")); + segmenter.tokenizePos("\u6211\u7231\u6E05\u534E\u5927\u5B66")); } /** @@ -140,9 +144,9 @@ void testMixedKnownAndUnknownRuns() { */ @Test void testSpansStayOriginalAfterLeadingWhitespace() { - final String text = " 我来到北京"; + final String text = " \u6211\u6765\u5230\u5317\u4EAC"; Assertions.assertArrayEquals( - new String[] {"我", "来到", "北京"}, + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC"}, segmenter.tokenize(text)); Assertions.assertArrayEquals(new Span[] { new Span(2, 3), new Span(3, 5), new Span(5, 7)}, From a256b90a61ea93ee98a45ff42044432afd19565a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:32:44 -0400 Subject: [PATCH 06/24] OPENNLP-1894: Document dictionary acquisition with a checksum-verifying download helper --- .../dev/README-mecab-dictionaries.md | 83 +++++++++++++++++++ .../dev/download-mecab-dictionary.sh | 69 +++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md create mode 100755 opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh diff --git a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md new file mode 100644 index 0000000000..c3244ba9c4 --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md @@ -0,0 +1,83 @@ + + +# CJK dictionaries for the lattice tokenizer + +The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and by using it you accept that dictionary's license. Read the license file inside the archive before use; nothing in this repository grants you those terms. + +## Known mecab-format dictionary projects + +| Dictionary | Language | Archive to download | Encoding of the files | +|---|---|---|---| +| IPADIC 2.7.0 | Japanese | `mecab-ipadic-2.7.0-20070801.tar.gz` from the MeCab project's SourceForge file area (`sourceforge.net/projects/mecab/files/mecab-ipadic/2.7.0-20070801/`) | EUC-JP | +| mecab-ko-dic 2.1.1 | Korean | `mecab-ko-dic-2.1.1-20180720.tar.gz` from the project's Bitbucket downloads page (`bitbucket.org/eunjeon/mecab-ko-dic/downloads/`) | UTF-8 | + +Both archives are gzip-compressed tars, the format `MecabDictionaryInstaller` reads. The encoding column matters: `MecabDictionary.load(Path)` assumes UTF-8, and a dictionary in any other encoding is loaded with the two-argument overload, for example `MecabDictionary.load(dir, Charset.forName("EUC-JP"))` for IPADIC. + +## Step 1: download the archive + +Either fetch the archive with any tool you like, or use the helper next to this file, which adds checksum verification: + +``` +./download-mecab-dictionary.sh ipadic.tar.gz [expected-sha256] +``` + +On the first run without a checksum the script prints the SHA-256 it computed; record it and pass it on later runs so a changed or corrupted download fails loudly instead of being installed. + +## Step 2: unpack with the installer + +The installer extracts only the files a `MecabDictionary` reads (`*.csv`, `*.def`, and `dicrc`), flattens them into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: + +```java +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.MecabDictionaryInstaller; + +int files = MecabDictionaryInstaller.install( + Path.of("ipadic.tar.gz").toUri(), Path.of("ipadic")); +``` + +The returned count is the number of dictionary files extracted. A remote URI works in the same call if you prefer to skip step 1 entirely and let the installer stream the download. + +## Step 3: load and tokenize + +```java +import java.nio.charset.Charset; +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.LatticeTokenizer; +import opennlp.tools.tokenize.lattice.MecabDictionary; + +MecabDictionary dictionary = + MecabDictionary.load(Path.of("ipadic"), Charset.forName("EUC-JP")); +LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary); +String[] tokens = tokenizer.tokenize("東京都に行く"); +``` + +For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` is enough. Loaded dictionaries and tokenizers are immutable and safe to share between threads, so load once and reuse. + +## Chinese: the unigram segmenter needs only a frequency lexicon + +`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use mecab dictionaries. It loads a plain text lexicon, one entry per line: the word, its count, and optionally a tag, separated by whitespace. Any word-frequency list you have the rights to use works: + +```java +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.UnigramSegmenter; + +UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); +String[] tokens = segmenter.tokenize("我来到北京天安门"); +``` + +As with the dictionaries, the lexicon's license is between you and its publisher; nothing is bundled. diff --git a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh b/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh new file mode 100755 index 0000000000..30c5b23bde --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Downloads a mecab-format dictionary archive for the lattice tokenizer. +# +# This script only fetches the archive and, when a checksum is given, verifies it. +# It does not unpack anything: unpacking is the job of +# opennlp.tools.tokenize.lattice.MecabDictionaryInstaller, which extracts exactly the +# files a MecabDictionary reads and nothing else. See README-mecab-dictionaries.md in +# this directory for the known dictionary projects, their encodings, and the Java +# steps that follow the download. +# +# The dictionary you download carries its own license, which you accept by using it. +# Nothing is bundled with Apache OpenNLP and no dictionary location is built in. + +set -euo pipefail + +usage() { + echo "usage: $0 [expected-sha256]" >&2 + echo "" >&2 + echo " archive-url where to fetch the gzip-compressed tar from" >&2 + echo " target-file where to store the downloaded archive" >&2 + echo " expected-sha256 optional checksum; when given, the download is verified" >&2 + echo " and a mismatch deletes the file and fails the script" >&2 + exit 2 +} + +[ $# -lt 2 ] && usage +url="$1" +target="$2" +expected="${3:-}" + +# Download to a temporary name first so an interrupted transfer never leaves a +# half-written file at the target path. +tmp="${target}.download" +echo "downloading ${url}" +curl --fail --location --retry 3 --output "${tmp}" "${url}" + +actual="$(sha256sum "${tmp}" | cut -d' ' -f1)" +if [ -n "${expected}" ]; then + if [ "${actual}" != "${expected}" ]; then + rm -f "${tmp}" + echo "checksum mismatch: expected ${expected}" >&2 + echo " but got ${actual}" >&2 + exit 1 + fi + echo "checksum verified: ${actual}" +else + echo "sha256 of the download: ${actual}" + echo "record this value and pass it as the third argument next time to verify" +fi + +mv "${tmp}" "${target}" +echo "stored ${target}" +echo "" +echo "next: unpack and load it from Java; see README-mecab-dictionaries.md" From 559fdd072322791376088bb23b497f3064399fbf Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 22:09:30 -0400 Subject: [PATCH 07/24] OPENNLP-1894: Categorize by code point, keep unknown candidates inside their category run, and validate context ids at load --- .../tokenize/lattice/LatticeTokenizer.java | 47 +++- .../tokenize/lattice/MecabDictionary.java | 263 +++++++++++++++--- .../lattice/LatticeTokenizerTest.java | 181 ++++++++++++ 3 files changed, 442 insertions(+), 49 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index cb610f74fe..70d9c774f9 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -212,16 +212,20 @@ private List candidates(String text, int from, int to, int offset) { }); final boolean lexiconMatch = matched[0]; - final Category category = dictionary.categoryOf(text.charAt(position)); + final int codePoint = text.codePointAt(position); + final Category category = dictionary.categoryOf(codePoint); if (!lexiconMatch || category.invoke()) { - int run = position + 1; - while (run < to - && dictionary.categoryOf(text.charAt(run)).name().equals(category.name())) { - run++; + int run = position + Character.charCount(codePoint); + while (run < to) { + final int next = text.codePointAt(run); + if (!dictionary.categoryOf(next).name().equals(category.name())) { + break; + } + run += Character.charCount(next); } final List templates = dictionary.unknownEntries(category.name()); if (templates != null) { - addUnknown(candidates, position, run, to, category, templates); + addUnknown(candidates, text, position, run, category, templates); } } if (candidates.isEmpty()) { @@ -230,7 +234,8 @@ private List candidates(String text, int from, int to, int offset) { final List fallback = dictionary.unknownEntries("DEFAULT"); if (fallback != null) { for (final WordEntry entry : fallback) { - candidates.add(new Node(position, position + 1, entry, true)); + candidates.add( + new Node(position, position + Character.charCount(codePoint), entry, true)); } } } @@ -241,22 +246,38 @@ private List candidates(String text, int from, int to, int offset) { return candidates; } - /** Emits unknown-word candidates per the category's grouping and length settings. */ - private static void addUnknown(List candidates, int position, int runEnd, - int to, Category category, List templates) { + /** + * Emits unknown-word candidates per the category's grouping and length settings. + * + *

Every candidate stays inside the same-category run, so an unknown word never + * glues characters of different categories together, and every length counts whole + * characters rather than code units.

+ * + * @param candidates Receives the candidates. + * @param text The text being segmented. + * @param position The position the candidates start at. + * @param runEnd The exclusive end of the same-category run starting at + * {@code position}. + * @param category The category of that run. + * @param templates The category's unknown-word templates. + */ + private static void addUnknown(List candidates, String text, int position, + int runEnd, Category category, List templates) { if (category.group()) { for (final WordEntry entry : templates) { candidates.add(new Node(position, runEnd, entry, true)); } } final int lengths = category.length(); - for (int length = 1; length <= lengths && position + length <= to; length++) { - if (category.group() && position + length == runEnd) { + int end = position; + for (int length = 1; length <= lengths && end < runEnd; length++) { + end += Character.charCount(text.codePointAt(end)); + if (category.group() && end == runEnd) { // This length coincides with the grouped run emitted above; skip the duplicate. continue; } for (final WordEntry entry : templates) { - candidates.add(new Node(position, position + length, entry, true)); + candidates.add(new Node(position, end, entry, true)); } } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 908bc75f2c..c8138d7cf4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -24,6 +24,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -63,6 +64,146 @@ private static final class TrieNode { private List entries; } + /** + * The {@code char.def} code point to category name mapping, over the whole Unicode + * code point range. + * + *

The Basic Multilingual Plane is held in a directly indexed array, which is one + * reference per BMP code point and is the entire mapping for a BMP-only dictionary. + * The supplementary planes are held as a sorted, non-overlapping range table searched + * by binary search, because dictionaries map them in a handful of large blocks: a + * directly indexed array over all of Unicode would spend more than four megabytes of + * references to say what a few range rows already say.

+ */ + private static final class CategoryTable { + + private final String[] bmp; + private final int[] rangeStart; + private final int[] rangeEnd; + private final String[] rangeCategory; + + private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, + String[] rangeCategory) { + this.bmp = bmp; + this.rangeStart = rangeStart; + this.rangeEnd = rangeEnd; + this.rangeCategory = rangeCategory; + } + + /** + * Looks up the category name a {@code char.def} mapping gives a code point. + * + * @param codePoint The code point to classify. + * @return The category name, or {@code null} when no mapping covers the code point. + */ + private String categoryOf(int codePoint) { + if (codePoint <= Character.MAX_VALUE) { + return bmp[codePoint]; + } + int low = 0; + int high = rangeStart.length - 1; + while (low <= high) { + final int middle = (low + high) >>> 1; + if (codePoint < rangeStart[middle]) { + high = middle - 1; + } else if (codePoint > rangeEnd[middle]) { + low = middle + 1; + } else { + return rangeCategory[middle]; + } + } + return null; + } + } + + /** + * Collects {@code char.def} mappings in file order and folds them into a + * {@link CategoryTable}, giving a later mapping precedence over an earlier one that + * covers the same code point, which is what direct indexing does for the BMP. + */ + private static final class CategoryTableBuilder { + + private final String[] bmp = new String[Character.MAX_VALUE + 1]; + private final List bounds = new ArrayList<>(); + private final List names = new ArrayList<>(); + + /** + * Records one inclusive code point range's category. + * + * @param from The first code point of the range. + * @param to The last code point of the range, inclusive. + * @param category The category name to give the range. Must not be {@code null}. + */ + private void map(int from, int to, String category) { + for (int c = from; c <= Math.min(to, Character.MAX_VALUE); c++) { + bmp[c] = category; + } + if (to > Character.MAX_VALUE) { + bounds.add(new int[] {Math.max(from, Character.MAX_VALUE + 1), to}); + names.add(category); + } + } + + /** + * Folds the recorded mappings into their lookup table. + * + * @return The table. Never {@code null}. + */ + private CategoryTable build() { + // Cut the supplementary ranges at every boundary they introduce, so that each + // resulting elementary interval is covered by a single winning range and the + // table stays sorted and non-overlapping for binary search. + final int[] edges = new int[bounds.size() * 2]; + for (int i = 0; i < bounds.size(); i++) { + edges[i * 2] = bounds.get(i)[0]; + edges[i * 2 + 1] = bounds.get(i)[1] + 1; + } + Arrays.sort(edges); + final List intervals = new ArrayList<>(); + final List categories = new ArrayList<>(); + for (int i = 0; i < edges.length - 1; i++) { + if (edges[i] == edges[i + 1]) { + continue; + } + final String winner = lastCovering(edges[i]); + if (winner == null) { + continue; + } + final int previous = intervals.size() - 1; + if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 + && categories.get(previous).equals(winner)) { + intervals.get(previous)[1] = edges[i + 1] - 1; + } else { + intervals.add(new int[] {edges[i], edges[i + 1] - 1}); + categories.add(winner); + } + } + final int[] starts = new int[intervals.size()]; + final int[] ends = new int[intervals.size()]; + for (int i = 0; i < intervals.size(); i++) { + starts[i] = intervals.get(i)[0]; + ends[i] = intervals.get(i)[1]; + } + return new CategoryTable(bmp, starts, ends, categories.toArray(new String[0])); + } + + /** + * Finds the category of the last recorded range covering a code point. + * + * @param codePoint The code point to look up. + * @return The category name, or {@code null} when no recorded range covers it. + */ + private String lastCovering(int codePoint) { + for (int i = bounds.size() - 1; i >= 0; i--) { + final int[] range = bounds.get(i); + if (codePoint >= range[0] && codePoint <= range[1]) { + return names.get(i); + } + } + return null; + } + } + /** Receives one common-prefix match during {@link #prefixMatches}. */ interface PrefixMatchConsumer { @@ -80,18 +221,18 @@ interface PrefixMatchConsumer { private final short[] connectionCosts; private final int rightSize; private final Map categories; - private final String[] categoryOfChar; + private final CategoryTable categoryTable; private final Map> unknownEntries; private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, short[] connectionCosts, int rightSize, Map categories, - String[] categoryOfChar, Map> unknownEntries) { + CategoryTable categoryTable, Map> unknownEntries) { this.lexicon = lexicon; this.maxSurfaceLength = maxSurfaceLength; this.connectionCosts = connectionCosts; this.rightSize = rightSize; this.categories = categories; - this.categoryOfChar = categoryOfChar; + this.categoryTable = categoryTable; this.unknownEntries = unknownEntries; } @@ -116,25 +257,17 @@ public static MecabDictionary load(Path directory) throws IOException { * @param charset The encoding the distribution uses, for example UTF-8 or EUC-JP. * Must not be {@code null}. * @return The loaded dictionary. Never {@code null}. - * @throws IOException Thrown if reading fails, a required file is missing, or a file - * is malformed. + * @throws IOException Thrown if reading fails, a required file is missing, a file is + * malformed, or a lexicon entry's context ids are outside the + * {@code matrix.def} dimensions. * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static MecabDictionary load(Path directory, Charset charset) throws IOException { if (directory == null || charset == null) { throw new IllegalArgumentException("directory and charset must not be null"); } - final Map> lexicon = new HashMap<>(); - int maxSurface = 0; - try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { - for (final Path csv : csvFiles) { - maxSurface = Math.max(maxSurface, readLexicon(csv, charset, lexicon)); - } - } - if (lexicon.isEmpty()) { - throw new IOException("no lexicon entries found under " + directory); - } - + // The connection matrix is read first because its dimensions are what every + // lexicon entry's context ids have to be inside of. final List matrixLines = readLines(directory.resolve("matrix.def"), charset); if (matrixLines.isEmpty()) { throw new IOException("empty matrix.def under " + directory); @@ -145,6 +278,10 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final int leftSize = parseInt(header[0], "matrix.def", 1); final int rightSize = parseInt(header[1], "matrix.def", 1); + if (leftSize < 1 || rightSize < 1) { + throw new IOException("matrix.def dimensions must be positive, got " + + leftSize + " " + rightSize); + } final short[] costs = new short[leftSize * rightSize]; for (int i = 1; i < matrixLines.size(); i++) { final String line = matrixLines.get(i).trim(); @@ -157,19 +294,36 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final int right = parseInt(fields[0], "matrix.def", i + 1); final int left = parseInt(fields[1], "matrix.def", i + 1); + if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { + throw new IOException("malformed matrix.def line " + (i + 1) + + ": context ids " + right + " " + left + + " are outside the declared dimensions " + leftSize + " " + rightSize); + } costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); } + final Map> lexicon = new HashMap<>(); + int maxSurface = 0; + try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { + for (final Path csv : csvFiles) { + maxSurface = Math.max(maxSurface, + readLexicon(csv, charset, lexicon, leftSize, rightSize)); + } + } + if (lexicon.isEmpty()) { + throw new IOException("no lexicon entries found under " + directory); + } + final Map categories = new HashMap<>(); - final String[] categoryOfChar = new String[Character.MAX_VALUE + 1]; + final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); readCharacterDefinition(directory.resolve("char.def"), charset, categories, - categoryOfChar); + categoryTable); final Map> unknown = new HashMap<>(); - readLexicon(directory.resolve("unk.def"), charset, unknown); + readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryOfChar, unknown); + categories, categoryTable.build(), unknown); } /** Folds the surface-keyed lexicon into a character trie for prefix search. */ @@ -186,9 +340,24 @@ private static TrieNode buildTrie(Map> lexicon) { return root; } - /** Reads one lexicon-format CSV file; returns the longest surface seen. */ + /** + * Reads one lexicon-format CSV file, rejecting any entry whose context ids the + * connection matrix cannot be indexed with. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @param target Receives the entries, keyed by surface form. + * @param leftSize The first {@code matrix.def} dimension, which bounds right context + * ids. + * @param rightSize The second {@code matrix.def} dimension, which bounds left context + * ids. + * @return The length in characters of the longest surface form read. + * @throws IOException Thrown if the file is missing, an entry is malformed, or an + * entry's context id is outside the matrix dimensions. + */ private static int readLexicon(Path file, Charset charset, - Map> target) throws IOException { + Map> target, int leftSize, int rightSize) + throws IOException { int maxSurface = 0; int lineNumber = 0; for (final String line : readLines(file, charset)) { @@ -204,9 +373,19 @@ private static int readLexicon(Path file, Charset charset, if (surface.isEmpty()) { continue; } - final WordEntry entry = new WordEntry( - parseInt(fields.get(1), file.toString(), lineNumber), - parseInt(fields.get(2), file.toString(), lineNumber), + final int leftId = parseInt(fields.get(1), file.toString(), lineNumber); + final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); + if (leftId < 0 || leftId >= rightSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": left context id " + leftId + " is outside the matrix.def dimensions " + + leftSize + " " + rightSize); + } + if (rightId < 0 || rightId >= leftSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": right context id " + rightId + " is outside the matrix.def dimensions " + + leftSize + " " + rightSize); + } + final WordEntry entry = new WordEntry(leftId, rightId, parseInt(fields.get(3), file.toString(), lineNumber), List.copyOf(fields.subList(4, fields.size()))); target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); @@ -217,7 +396,8 @@ private static int readLexicon(Path file, Charset charset, /** Reads char.def: category behavior lines and code point mapping lines. */ private static void readCharacterDefinition(Path file, Charset charset, - Map categories, String[] categoryOfChar) throws IOException { + Map categories, CategoryTableBuilder categoryTable) + throws IOException { int lineNumber = 0; for (final String raw : readLines(file, charset)) { lineNumber++; @@ -240,9 +420,11 @@ private static void readCharacterDefinition(Path file, Charset charset, if (fields.length < 2) { throw new IOException("mapping without category at " + file + " line " + lineNumber); } - for (int c = from; c <= to && c <= Character.MAX_VALUE; c++) { - categoryOfChar[c] = fields[1]; + if (from > to) { + throw new IOException("code point range descends at " + file + " line " + + lineNumber); } + categoryTable.map(from, to, fields[1]); } else { if (fields.length < 4) { throw new IOException("malformed category at " + file + " line " + lineNumber); @@ -310,13 +492,16 @@ int connectionCost(int rightId, int leftId) { } /** - * Classifies a character. + * Classifies a character by code point, so that a character outside the Basic + * Multilingual Plane is classified as the one character it is rather than as its two + * surrogates. * - * @param c The character. - * @return Its category, falling back to {@code DEFAULT}. Never {@code null}. + * @param codePoint The code point to classify. + * @return Its category, falling back to {@code DEFAULT} when no {@code char.def} + * mapping covers the code point. Never {@code null}. */ - Category categoryOf(char c) { - final String name = categoryOfChar[c]; + Category categoryOf(int codePoint) { + final String name = categoryTable.categoryOf(codePoint); final Category category = name == null ? null : categories.get(name); return category != null ? category : categories.get("DEFAULT"); } @@ -436,15 +621,21 @@ private static int parseInt(String text, String file, int lineNumber) * @param text The field text including the {@code 0x} prefix. * @param file The file being read, for the error message. * @param lineNumber The line being read, for the error message. - * @return The parsed code point. - * @throws IOException Thrown if the field is not a valid hexadecimal code point. + * @return The parsed code point, which may be in a supplementary plane. + * @throws IOException Thrown if the field is not a valid hexadecimal code point or + * names a value no Unicode code point has. */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { + final int codePoint; try { - return Integer.parseInt(text.trim().substring(2), 16); + codePoint = Integer.parseInt(text.trim().substring(2), 16); } catch (RuntimeException e) { throw new IOException("malformed code point in " + file + " line " + lineNumber, e); } + if (!Character.isValidCodePoint(codePoint)) { + throw new IOException("code point out of range in " + file + " line " + lineNumber); + } + return codePoint; } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 83b823706c..bf7a5f8c39 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -119,6 +119,37 @@ void testUnknownKanjiPreferOneMorphemeOverTwo() { Assertions.assertEquals(true, morphemes.get(0).unknown()); } + /** + * Verifies that an unknown-word candidate never spans a character category boundary. + * An unlisted kanji directly followed by a Latin letter must be analyzed as two + * morphemes of their own categories, never as one KANJI morpheme whose surface glues + * the kanji to the letter. + */ + @Test + void testUnknownCandidatesNeverSpanCategoryBoundaries() { + final String text = "\u5CE0a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0", "a"}, tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1), new Span(1, 2)}, + tokenizer.tokenizePos(text)); + final List morphemes = tokenizer.analyze(text); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(1).features()); + } + + /** + * Verifies that bounding unknown-word candidates by the category run does not under + * generate inside the run: a two-kanji unlisted run followed by a Latin letter still + * offers the length-two KANJI candidate, which wins over two single-kanji morphemes. + */ + @Test + void testUnknownRunStillOffersWithinCategoryLengths() { + final String text = "\u5CE0\u9053a"; + Assertions.assertArrayEquals(new String[] {"\u5CE0\u9053", "a"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + tokenizer.tokenizePos(text)); + } + @Test void testWhitespaceSeparatesAndIsNeverAMorpheme() { final String text = "\u6771\u4EAC \u306B \u884C\u304F"; @@ -216,6 +247,8 @@ void testSpansStayOriginalAfterLeadingWhitespace() { */ @Test void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the short row is what load rejects. + writeUnitMatrixDictionary(broken); Files.write(broken.resolve("lexicon.csv"), "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); @@ -227,6 +260,8 @@ void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { */ @Test void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { + // The rest of the dictionary is well formed, so the cost column is what load rejects. + writeUnitMatrixDictionary(broken); Files.write(broken.resolve("lexicon.csv"), "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); @@ -295,6 +330,152 @@ void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } + /** + * Writes a miniature dictionary whose {@code char.def} maps a supplementary plane + * range, the shape a UniDic-style distribution uses for the CJK extension blocks. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeSupplementaryDictionary(Path target) throws IOException { + Files.write(target.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun,common\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x4E00..0x9FFF KANJI", + "0x20000..0x2A6DF KANJI", + "0x0061..0x007A LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("unk.def"), String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "LATIN,0,0,4000,noun,foreign", + "").getBytes(StandardCharsets.UTF_8)); + } + + /** + * Verifies that a {@code char.def} range above U+FFFF is honored rather than + * discarded: a supplementary plane ideograph inside the mapped range takes the + * category the range names, while a supplementary code point outside every mapped + * range still falls back to DEFAULT. + */ + @Test + void testSupplementaryCharDefRangeIsHonored(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final MecabDictionary dictionary = MecabDictionary.load(supplementary); + // U+20BB7 is a CJK extension B ideograph inside the mapped range. + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20BB7).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x6771).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2460).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x2A6E0).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf('a').name()); + } + + /** + * Verifies that a supplementary plane ideograph is analyzed as the single character + * it is: one morpheme whose span covers both code units and which carries the + * features of the category its {@code char.def} range names, never one morpheme per + * surrogate. The second case shows the category's length templates count characters, + * not code units, so a run of two supplementary ideographs is still reachable by the + * length-two template. + */ + @Test + void testSupplementaryIdeographIsOneMorpheme(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + // U+20BB7 written as its surrogate pair, per this file's ASCII-only convention. + final String text = "\uD842\uDFB7"; + final List morphemes = supplementaryTokenizer.analyze(text); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals(text, morphemes.get(0).surface()); + Assertions.assertEquals(new Span(0, 2), morphemes.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), morphemes.get(0).features()); + + final List pair = supplementaryTokenizer.analyze(text + text); + Assertions.assertEquals(1, pair.size()); + Assertions.assertEquals(new Span(0, 4), pair.get(0).span()); + Assertions.assertEquals(List.of("noun", "unknown"), pair.get(0).features()); + } + + /** + * Verifies that a supplementary plane ideograph does not absorb neighbouring text of + * another category: the ideograph and an unmapped symbol beside it stay two + * morphemes, each span covering whole characters. + */ + @Test + void testSupplementaryIdeographDoesNotAbsorbItsNeighbour(@TempDir Path supplementary) + throws IOException { + writeSupplementaryDictionary(supplementary); + final LatticeTokenizer supplementaryTokenizer = + new LatticeTokenizer(MecabDictionary.load(supplementary)); + final String text = "\uD842\uDFB7\u2460"; + Assertions.assertArrayEquals(new String[] {"\uD842\uDFB7", "\u2460"}, + supplementaryTokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] {new Span(0, 2), new Span(2, 3)}, + supplementaryTokenizer.tokenizePos(text)); + } + + /** + * Writes every dictionary file except the lexicon, so a test can supply a lexicon of + * its own against a one by one connection matrix. + * + * @param target The directory to write the dictionary files into. Must not be + * {@code null} and must exist. + * @throws IOException Thrown if writing any of the files fails. + */ + private static void writeUnitMatrixDictionary(Path target) throws IOException { + Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("char.def"), + "DEFAULT 0 1 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(target.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + } + + /** + * Verifies that a lexicon row whose right context id is outside the + * {@code matrix.def} dimensions is rejected at load time, naming the file, the line, + * and the offending id, rather than reaching the cost matrix with an out of range + * index during segmentation. + */ + @Test + void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + Files.write(mismatched.resolve("lexicon.csv"), + "\u6771,0,5,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + + " line 1: right context id 5 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + + /** + * Verifies that a lexicon row whose left context id is outside the {@code matrix.def} + * dimensions is rejected at load time, naming the file, the line, and the offending + * id. + */ + @Test + void testLeftContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) + throws IOException { + writeUnitMatrixDictionary(mismatched); + Files.write(mismatched.resolve("lexicon.csv"), + "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(mismatched)); + Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + + " line 2: left context id 7 is outside the matrix.def dimensions 1 1", + e.getMessage()); + } + @Test void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, From 04d380ce77a8af34f54e02e5a0fd030e972fc3c1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 00:53:57 -0400 Subject: [PATCH 08/24] OPENNLP-1894: Precompute category runs, unbox the trie, and reject inexpressible dictionary values The lattice tokenizer rescanned the same-category run from every position, so a run of L characters cost on the order of L squared category lookups; a 16,000-character katakana run measured around half a second. One right-to-left pass per stretch now fixes every position's category and run end, and the same 16,000-character run tokenizes in about half a millisecond at 31 million characters per second. The character table holds Category instances instead of names, so the per-character path compares by identity with no name-map lookup, and a char.def mapping to an undefined category now fails at load naming the code point. The lexicon trie's children are sorted character arrays found by binary search, so a descent no longer boxes a Character per step. matrix.def loading rejects connection costs outside the 16-bit range instead of silently truncating them, and dimension products beyond the addressable array size fail at the header. The unigram segmenter's unknown-character fallback advances one code point, never one code unit, so an unknown supplementary character is stepped over whole and no span can split its surrogate halves. --- .../tokenize/lattice/LatticeTokenizer.java | 66 ++++++-- .../tokenize/lattice/MecabDictionary.java | 141 ++++++++++++++---- .../tokenize/lattice/UnigramSegmenter.java | 11 +- .../lattice/LatticeTokenizerTest.java | 138 +++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 24 +++ 5 files changed, 337 insertions(+), 43 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 70d9c774f9..f7a479bf83 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -143,11 +143,19 @@ private void decode(String text, int from, int to, List morphemes) { final boolean[] reachable = new boolean[length + 1]; reachable[0] = true; + // One right-to-left pass fixes each position's category and same-category run end, + // so candidate generation reads them instead of rescanning the run from every + // position, which would cost the square of the run length on long uniform runs. + final Category[] categoryAt = new Category[length]; + final int[] runEndAt = new int[length]; + computeCategoryRuns(text, from, to, categoryAt, runEndAt); + for (int i = 0; i < length; i++) { if (!reachable[i]) { continue; } - final List candidates = candidates(text, from, to, i); + final List candidates = + candidates(text, from, to, i, categoryAt[i], runEndAt[i]); for (final Node candidate : candidates) { relax(candidate, i == 0 ? null : endingAt.get(i)); if (candidate.pathCost < Long.MAX_VALUE) { @@ -199,8 +207,39 @@ private void relax(Node candidate, List predecessors) { } } + /** + * Fills the per-position category and same-category run end for one stretch, in one + * right-to-left pass over its code points. Positions inside a surrogate pair keep a + * {@code null} category; no candidate ever starts there. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param categoryAt Receives each position's category, indexed by {@code + * position - from}. + * @param runEndAt Receives each position's exclusive same-category run end, indexed + * the same way. + */ + private void computeCategoryRuns(String text, int from, int to, + Category[] categoryAt, int[] runEndAt) { + int next = -1; + for (int position = to; position > from; ) { + final int codePoint = text.codePointBefore(position); + position -= Character.charCount(codePoint); + final int index = position - from; + categoryAt[index] = dictionary.categoryOf(codePoint); + if (next >= 0 && categoryAt[next] == categoryAt[index]) { + runEndAt[index] = runEndAt[next]; + } else { + runEndAt[index] = next >= 0 ? next + from : to; + } + next = index; + } + } + /** Gathers lexicon matches and unknown-word candidates starting at one position. */ - private List candidates(String text, int from, int to, int offset) { + private List candidates(String text, int from, int to, int offset, + Category positionCategory, int positionRunEnd) { final int position = from + offset; final List candidates = new ArrayList<>(); final boolean[] matched = new boolean[1]; @@ -213,19 +252,22 @@ private List candidates(String text, int from, int to, int offset) { final boolean lexiconMatch = matched[0]; final int codePoint = text.codePointAt(position); - final Category category = dictionary.categoryOf(codePoint); + final Category category; + final int runEnd; + if (positionCategory == null) { + // Only a lexicon surface ending inside a surrogate pair could make such a + // position reachable; classify the stray code unit on the spot so the lattice + // stays connected the way it always did. + category = dictionary.categoryOf(codePoint); + runEnd = position + Character.charCount(codePoint); + } else { + category = positionCategory; + runEnd = positionRunEnd; + } if (!lexiconMatch || category.invoke()) { - int run = position + Character.charCount(codePoint); - while (run < to) { - final int next = text.codePointAt(run); - if (!dictionary.categoryOf(next).name().equals(category.name())) { - break; - } - run += Character.charCount(next); - } final List templates = dictionary.unknownEntries(category.name()); if (templates != null) { - addUnknown(candidates, text, position, run, category, templates); + addUnknown(candidates, text, position, runEnd, category, templates); } } if (candidates.isEmpty()) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index c8138d7cf4..4a34e24d27 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -58,10 +58,56 @@ record WordEntry(int leftId, int rightId, int cost, List features) { record Category(String name, boolean invoke, boolean group, int length) { } - /** One node of the lexicon trie, keyed by the next surface character. */ + /** + * One frozen node of the lexicon trie: children are held as a sorted character + * array with a parallel node array and found by binary search, so a descent never + * boxes the character the way a map keyed by {@link Character} would on every step + * of the innermost matching loop. + */ private static final class TrieNode { - private final Map children = new HashMap<>(); + + private final char[] keys; + private final TrieNode[] nodes; + private final List entries; + + private TrieNode(char[] keys, TrieNode[] nodes, List entries) { + this.keys = keys; + this.nodes = nodes; + this.entries = entries; + } + + /** + * Descends one character. + * + * @param c The next surface character. + * @return The child node, or {@code null} when no surface continues with {@code c}. + */ + private TrieNode child(char c) { + final int index = Arrays.binarySearch(keys, c); + return index >= 0 ? nodes[index] : null; + } + } + + /** One mutable trie node during construction, frozen into a {@link TrieNode}. */ + private static final class TrieBuilderNode { + + private final Map children = new HashMap<>(); private List entries; + + /** Freezes this node and its subtree into the sorted-array form. */ + private TrieNode freeze() { + final char[] keys = new char[children.size()]; + int i = 0; + for (final Character key : children.keySet()) { + keys[i++] = key; + } + Arrays.sort(keys); + final TrieNode[] nodes = new TrieNode[keys.length]; + for (int k = 0; k < keys.length; k++) { + nodes[k] = children.get(keys[k]).freeze(); + } + return new TrieNode(keys, nodes, entries); + } } /** @@ -77,13 +123,13 @@ private static final class TrieNode { */ private static final class CategoryTable { - private final String[] bmp; + private final Category[] bmp; private final int[] rangeStart; private final int[] rangeEnd; - private final String[] rangeCategory; + private final Category[] rangeCategory; - private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, - String[] rangeCategory) { + private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, + Category[] rangeCategory) { this.bmp = bmp; this.rangeStart = rangeStart; this.rangeEnd = rangeEnd; @@ -91,12 +137,16 @@ private CategoryTable(String[] bmp, int[] rangeStart, int[] rangeEnd, } /** - * Looks up the category name a {@code char.def} mapping gives a code point. + * Looks up the category a {@code char.def} mapping gives a code point. The table + * holds the {@link Category} instances themselves, so a lookup on the tokenizer's + * per-character path costs one array read or one binary search, never a name map + * access, and two code points of one category share one instance to compare by + * identity. * * @param codePoint The code point to classify. - * @return The category name, or {@code null} when no mapping covers the code point. + * @return The category, or {@code null} when no mapping covers the code point. */ - private String categoryOf(int codePoint) { + private Category categoryOf(int codePoint) { if (codePoint <= Character.MAX_VALUE) { return bmp[codePoint]; } @@ -149,7 +199,7 @@ private void map(int from, int to, String category) { * * @return The table. Never {@code null}. */ - private CategoryTable build() { + private CategoryTable build(Map categories) throws IOException { // Cut the supplementary ranges at every boundary they introduce, so that each // resulting elementary interval is covered by a single winning range and the // table stays sorted and non-overlapping for binary search. @@ -160,7 +210,7 @@ private CategoryTable build() { } Arrays.sort(edges); final List intervals = new ArrayList<>(); - final List categories = new ArrayList<>(); + final List winners = new ArrayList<>(); for (int i = 0; i < edges.length - 1; i++) { if (edges[i] == edges[i + 1]) { continue; @@ -171,11 +221,11 @@ private CategoryTable build() { } final int previous = intervals.size() - 1; if (previous >= 0 && intervals.get(previous)[1] == edges[i] - 1 - && categories.get(previous).equals(winner)) { + && winners.get(previous).equals(winner)) { intervals.get(previous)[1] = edges[i + 1] - 1; } else { intervals.add(new int[] {edges[i], edges[i + 1] - 1}); - categories.add(winner); + winners.add(winner); } } final int[] starts = new int[intervals.size()]; @@ -184,7 +234,32 @@ private CategoryTable build() { starts[i] = intervals.get(i)[0]; ends[i] = intervals.get(i)[1]; } - return new CategoryTable(bmp, starts, ends, categories.toArray(new String[0])); + final Category[] resolvedBmp = new Category[bmp.length]; + for (int c = 0; c < bmp.length; c++) { + if (bmp[c] != null) { + resolvedBmp[c] = resolve(bmp[c], categories, c); + } + } + final Category[] resolvedRanges = new Category[winners.size()]; + for (int i = 0; i < winners.size(); i++) { + resolvedRanges[i] = resolve(winners.get(i), categories, starts[i]); + } + return new CategoryTable(resolvedBmp, starts, ends, resolvedRanges); + } + + /** + * Resolves a mapped category name against the defined categories, so a mapping to + * a name the {@code char.def} category section never defined fails at load with + * the offending code point instead of falling back silently at lookup time. + */ + private static Category resolve(String name, Map categories, + int codePoint) throws IOException { + final Category category = categories.get(name); + if (category == null) { + throw new IOException(String.format( + "char.def maps U+%04X to the undefined category %s", codePoint, name)); + } + return category; } /** @@ -222,6 +297,7 @@ interface PrefixMatchConsumer { private final int rightSize; private final Map categories; private final CategoryTable categoryTable; + private final Category defaultCategory; private final Map> unknownEntries; private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, @@ -233,6 +309,7 @@ private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, this.rightSize = rightSize; this.categories = categories; this.categoryTable = categoryTable; + this.defaultCategory = categories.get("DEFAULT"); this.unknownEntries = unknownEntries; } @@ -282,7 +359,12 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc throw new IOException("matrix.def dimensions must be positive, got " + leftSize + " " + rightSize); } - final short[] costs = new short[leftSize * rightSize]; + final long cells = (long) leftSize * rightSize; + if (cells > Integer.MAX_VALUE) { + throw new IOException("matrix.def dimensions " + leftSize + " x " + rightSize + + " overflow the addressable connection matrix"); + } + final short[] costs = new short[(int) cells]; for (int i = 1; i < matrixLines.size(); i++) { final String line = matrixLines.get(i).trim(); if (line.isEmpty()) { @@ -299,7 +381,13 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc + ": context ids " + right + " " + left + " are outside the declared dimensions " + leftSize + " " + rightSize); } - costs[right * rightSize + left] = (short) parseInt(fields[2], "matrix.def", i + 1); + final int cost = parseInt(fields[2], "matrix.def", i + 1); + if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { + throw new IOException("malformed matrix.def line " + (i + 1) + + ": connection cost " + cost + " is outside the 16-bit range the" + + " format defines"); + } + costs[right * rightSize + left] = (short) cost; } final Map> lexicon = new HashMap<>(); @@ -318,26 +406,26 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); readCharacterDefinition(directory.resolve("char.def"), charset, categories, categoryTable); - final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryTable.build(), unknown); + categories, categoryTable.build(categories), unknown); } /** Folds the surface-keyed lexicon into a character trie for prefix search. */ private static TrieNode buildTrie(Map> lexicon) { - final TrieNode root = new TrieNode(); + final TrieBuilderNode root = new TrieBuilderNode(); for (final Map.Entry> entry : lexicon.entrySet()) { - TrieNode node = root; + TrieBuilderNode node = root; final String surface = entry.getKey(); for (int i = 0; i < surface.length(); i++) { - node = node.children.computeIfAbsent(surface.charAt(i), key -> new TrieNode()); + node = node.children.computeIfAbsent(surface.charAt(i), + key -> new TrieBuilderNode()); } node.entries = List.copyOf(entry.getValue()); } - return root; + return root.freeze(); } /** @@ -448,7 +536,7 @@ private static void readCharacterDefinition(Path file, Charset charset, List lookup(String surface) { TrieNode node = lexicon; for (int i = 0; i < surface.length() && node != null; i++) { - node = node.children.get(surface.charAt(i)); + node = node.child(surface.charAt(i)); } return node == null ? null : node.entries; } @@ -465,7 +553,7 @@ List lookup(String surface) { void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { TrieNode node = lexicon; for (int i = from; i < to; i++) { - node = node.children.get(text.charAt(i)); + node = node.child(text.charAt(i)); if (node == null) { return; } @@ -501,9 +589,8 @@ int connectionCost(int rightId, int leftId) { * mapping covers the code point. Never {@code null}. */ Category categoryOf(int codePoint) { - final String name = categoryTable.categoryOf(codePoint); - final Category category = name == null ? null : categories.get(name); - return category != null ? category : categories.get("DEFAULT"); + final Category category = categoryTable.categoryOf(codePoint); + return category != null ? category : defaultCategory; } /** diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index ea3d188bc5..b4cfaa8ea6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -218,11 +218,14 @@ private void decode(String text, int from, int to, List spans) { continue; } // A single-character step at the unknown log-probability keeps every position - // reachable even where no lexicon word matches. + // reachable even where no lexicon word matches. The step advances one code + // point, never one code unit, so an unknown supplementary character is stepped + // over whole and no span boundary can land between its surrogate halves. + final int width = Character.charCount(text.codePointAt(from + i)); final double fallback = best[i] + unknownLogProbability; - if (fallback > best[i + 1]) { - best[i + 1] = fallback; - previous[i + 1] = i; + if (i + width <= length && fallback > best[i + width]) { + best[i + width] = fallback; + previous[i + width] = i; } WordTrie node = trie; for (int j = from + i; j < to; j++) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index bf7a5f8c39..e1be5a4a98 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -484,4 +484,142 @@ void testInvalidArguments() { () -> MecabDictionary.load(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); } + + /** + * Verifies the supplementary range table's interval cutting and precedence: a later + * {@code char.def} mapping strictly inside an earlier one wins exactly on its own + * stretch, and the earlier category resumes after it, so the cut produces three + * intervals from two overlapping ranges. + */ + @Test + void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlapped) + throws IOException { + Files.write(overlapped.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "LATIN 1 1 0", + "", + "0x20000..0x2FFFF KANJI", + "0x24000..0x25000 LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(overlapped.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final MecabDictionary dictionary = MecabDictionary.load(overlapped); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x23FFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x24000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x25000).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x25001).name()); + Assertions.assertEquals("KANJI", dictionary.categoryOf(0x2FFFF).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x30000).name()); + } + + /** + * Verifies a {@code char.def} range straddling the BMP boundary: the part up to + * U+FFFF lands in the directly indexed table and the rest in the range table, and + * both halves answer the same category with no gap at the seam. + */ + @Test + void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) + throws IOException { + Files.write(straddling.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "LATIN 1 1 0", + "", + "0xFF00..0x10040 LATIN", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(straddling.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final MecabDictionary dictionary = MecabDictionary.load(straddling); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFF00).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFFFF).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10000).name()); + Assertions.assertEquals("LATIN", dictionary.categoryOf(0x10040).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0x10041).name()); + Assertions.assertEquals("DEFAULT", dictionary.categoryOf(0xFEFF).name()); + } + + /** + * Verifies that a {@code char.def} mapping to a category its category section never + * defined fails at load, naming the code point and the ghost category, instead of + * silently falling back to DEFAULT at lookup time. + */ + @Test + void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOException { + Files.write(ghost.resolve("lexicon.csv"), + "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("matrix.def"), + "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("char.def"), String.join("\n", + "DEFAULT 0 1 0", + "", + "0x0100..0x0110 GHOST", + "").getBytes(StandardCharsets.UTF_8)); + Files.write(ghost.resolve("unk.def"), + "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(ghost)); + Assertions.assertEquals("char.def maps U+0100 to the undefined category GHOST", + e.getMessage()); + } + + /** + * Verifies that a connection cost outside the 16-bit range the binary matrix format + * defines is rejected at load instead of being truncated by the narrowing cast into + * a silently different cost. + */ + @Test + void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "1 1\n0 0 40000\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: connection cost 40000 is" + + " outside the 16-bit range the format defines", e.getMessage()); + } + + /** + * Verifies that {@code matrix.def} dimensions whose product exceeds the addressable + * array size fail loud at the header instead of overflowing the int multiplication + * into a negative or wrapped allocation size. + */ + @Test + void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "70000 70000\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("matrix.def dimensions 70000 x 70000 overflow the" + + " addressable connection matrix", e.getMessage()); + } + + /** + * Verifies that a {@code matrix.def} data row naming context ids outside the + * declared dimensions is rejected at load with the offending line and ids. + */ + @Test + void testMatrixRowContextIdsOutsideDimensionsFailLoud(@TempDir Path broken) + throws IOException { + writeUnitMatrixDictionary(broken); + Files.write(broken.resolve("matrix.def"), + "1 1\n2 0 5\n".getBytes(StandardCharsets.UTF_8)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("malformed matrix.def line 2: context ids 2 0 are outside" + + " the declared dimensions 1 1", e.getMessage()); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index 9725b8c729..e6ebd604e1 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -176,4 +176,28 @@ void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, () -> segmenter.tokenizePos(null)); } + + /** + * Verifies that the unknown-character fallback advances one code point, never one + * code unit: a supplementary character absent from the lexicon comes back as one + * span over both of its surrogate halves, and no span boundary ever lands between + * them. + */ + @org.junit.jupiter.api.Test + void testUnknownSupplementaryCharacterIsNeverSplit() { + // U+20BB7, a CJK extension B ideograph, written as its surrogate pair + final String text = "\uD842\uDFB7\uD842\uDFB7"; + final opennlp.tools.util.Span[] spans = segmenter.tokenizePos(text); + for (final opennlp.tools.util.Span span : spans) { + Assertions.assertEquals(0, span.getStart() % 2, + "span must start on a code point boundary: " + span); + Assertions.assertEquals(0, span.getEnd() % 2, + "span must end on a code point boundary: " + span); + } + int covered = 0; + for (final opennlp.tools.util.Span span : spans) { + covered += span.length(); + } + Assertions.assertEquals(text.length(), covered); + } } From 625452d927e83654afb67767622e40b44af229dc Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 06:54:45 -0400 Subject: [PATCH 09/24] OPENNLP-1894: Hold the lexicon in a double-array trie with frequency-recoded labels The lexicon trie's per-node child lookup, a binary search over the node's fan-out, paid about a dozen comparisons at the root of a real dictionary; the classic base/check double-array makes every transition one array read and one comparison. Characters are recoded into dense labels ordered by descending frequency before the array is built, so the array stays compact although CJK surfaces draw on tens of thousands of distinct characters, and a character the lexicon never uses misses in the recode table before the array is consulted. On the IPADIC harness the prefix walk now matches the fastest previous implementation at 5.6M chars/s with strictly constant-time transitions, and building the array adds about a quarter second to the 392k-entry load. --- .../tokenize/lattice/MecabDictionary.java | 304 ++++++++++++++---- 1 file changed, 236 insertions(+), 68 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 4a34e24d27..cd1f46ba7f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -59,54 +59,250 @@ record Category(String name, boolean invoke, boolean group, int length) { } /** - * One frozen node of the lexicon trie: children are held as a sorted character - * array with a parallel node array and found by binary search, so a descent never - * boxes the character the way a map keyed by {@link Character} would on every step - * of the innermost matching loop. + * The lexicon as a double-array trie: one transition is one array read and one + * comparison, so the per-position prefix walk of the tokenizer never hashes, never + * binary-searches a fan-out, and never boxes. Characters are recoded into dense + * labels ordered by frequency before the array is built, which keeps the array + * compact although CJK surfaces draw from tens of thousands of distinct + * characters; a character the lexicon never uses misses in the recode table before + * the array is even consulted. + * + *

The layout is the classic base/check pair: from state {@code s}, label + * {@code c} leads to {@code t = base[s] + c} exactly when {@code check[t] == s}. + * Label {@code 0} terminates a surface and leads to a state whose negative base + * encodes the index of the surface's entry list.

*/ - private static final class TrieNode { + private static final class DoubleArrayLexicon { + + private final int[] base; + private final int[] check; + private final int[] codeOf; + private final List[] values; + + private DoubleArrayLexicon(int[] base, int[] check, int[] codeOf, + List[] values) { + this.base = base; + this.check = check; + this.codeOf = codeOf; + this.values = values; + } + + /** + * Builds the trie from the surface-keyed lexicon. + * + * @param lexicon The entries keyed by surface form. + * @return The built trie. Never {@code null}. + */ + @SuppressWarnings("unchecked") + private static DoubleArrayLexicon build(Map> lexicon) { + final String[] surfaces = lexicon.keySet().toArray(new String[0]); + Arrays.sort(surfaces); + final List[] values = new List[surfaces.length]; + for (int i = 0; i < surfaces.length; i++) { + values[i] = List.copyOf(lexicon.get(surfaces[i])); + } - private final char[] keys; - private final TrieNode[] nodes; - private final List entries; + // Dense recode: labels ordered by descending frequency get the small codes, so + // busy transitions cluster at the low end of the array. + final int[] frequency = new int[Character.MAX_VALUE + 1]; + for (final String surface : surfaces) { + for (int i = 0; i < surface.length(); i++) { + frequency[surface.charAt(i)]++; + } + } + final Integer[] chars = new Integer[Character.MAX_VALUE + 1]; + int distinct = 0; + for (int c = 0; c <= Character.MAX_VALUE; c++) { + if (frequency[c] > 0) { + chars[distinct++] = c; + } + } + final Integer[] ordered = Arrays.copyOf(chars, distinct); + Arrays.sort(ordered, (a, b) -> frequency[b] - frequency[a]); + final int[] codeOf = new int[Character.MAX_VALUE + 1]; + Arrays.fill(codeOf, -1); + for (int rank = 0; rank < ordered.length; rank++) { + codeOf[ordered[rank]] = rank + 1; + } - private TrieNode(char[] keys, TrieNode[] nodes, List entries) { - this.keys = keys; - this.nodes = nodes; - this.entries = entries; + final Builder builder = new Builder(surfaces, codeOf); + builder.insert(0, surfaces.length, 0, Builder.ROOT); + return new DoubleArrayLexicon(Arrays.copyOf(builder.base, builder.high + 1), + Arrays.copyOf(builder.check, builder.high + 1), codeOf, values); } /** - * Descends one character. + * Reports every surface starting at a text position, walking the array once. * - * @param c The next surface character. - * @return The child node, or {@code null} when no surface continues with {@code c}. + * @param text The text being segmented. + * @param from The position surfaces must start at. + * @param to The exclusive end of the searchable stretch. + * @param consumer Receives each match length with its entries. */ - private TrieNode child(char c) { - final int index = Arrays.binarySearch(keys, c); - return index >= 0 ? nodes[index] : null; + private void prefixMatches(String text, int from, int to, + PrefixMatchConsumer consumer) { + int state = Builder.ROOT; + for (int i = from; i < to; i++) { + final char c = text.charAt(i); + final int code = codeOf[c]; + if (code < 0) { + return; + } + final int next = base[state] + code; + if (next >= check.length || check[next] != state) { + return; + } + state = next; + final int terminal = base[state]; + if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { + consumer.accept(i - from + 1, values[-base[terminal] - 1]); + } + } + } + + /** + * Looks up the entries of an exact surface form. + * + * @param surface The surface form. + * @return The entries, or {@code null} when the surface is not listed. + */ + private List lookup(String surface) { + int state = Builder.ROOT; + for (int i = 0; i < surface.length(); i++) { + final int code = codeOf[surface.charAt(i)]; + if (code < 0) { + return null; + } + final int next = base[state] + code; + if (next >= check.length || check[next] != state) { + return null; + } + state = next; + } + final int terminal = base[state]; + if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { + return values[-base[terminal] - 1]; + } + return null; } - } - /** One mutable trie node during construction, frozen into a {@link TrieNode}. */ - private static final class TrieBuilderNode { + /** + * The recursive sorted-range builder: each call places one node's children by + * finding a base at which every child label lands on a free slot, then recurses + * per child range. A moving watermark keeps the free-slot search near-linear over + * real lexicons. + */ + private static final class Builder { + + private static final int ROOT = 1; + private static final int EMPTY = -1; + + private final String[] surfaces; + private final int[] codeOf; + private int[] base; + private int[] check; + private int high = ROOT; + private int watermark = ROOT + 1; + private int valueIndex; + + private Builder(String[] surfaces, int[] codeOf) { + this.surfaces = surfaces; + this.codeOf = codeOf; + base = new int[1 << 16]; + check = new int[1 << 16]; + Arrays.fill(check, EMPTY); + } - private final Map children = new HashMap<>(); - private List entries; + /** + * Places the children of one trie node. + * + * @param left The first surface of the node's range. + * @param right The exclusive last surface of the node's range. + * @param depth The character depth of the node. + * @param state The node's own slot. + */ + private void insert(int left, int right, int depth, int state) { + // gather the distinct child labels of this range, terminator first + final int[] labels = new int[right - left]; + int labelCount = 0; + int previous = -2; + for (int k = left; k < right; k++) { + final int label = surfaces[k].length() == depth + ? 0 : codeOf[surfaces[k].charAt(depth)]; + if (label != previous) { + labels[labelCount++] = label; + previous = label; + } + } + final int found = findBase(labels, labelCount); + base[state] = found; + for (int k = 0; k < labelCount; k++) { + final int child = found + labels[k]; + check[child] = state; + if (child > high) { + high = child; + } + } + // recurse over each child's sub-range + int start = left; + for (int k = 0; k < labelCount; k++) { + final int label = labels[k]; + int end = start; + while (end < right && (surfaces[end].length() == depth + ? 0 : codeOf[surfaces[end].charAt(depth)]) == label) { + end++; + } + final int child = found + label; + if (label == 0) { + base[child] = -(++valueIndex); + } else { + insert(start, end, depth + 1, child); + } + start = end; + } + } - /** Freezes this node and its subtree into the sorted-array form. */ - private TrieNode freeze() { - final char[] keys = new char[children.size()]; - int i = 0; - for (final Character key : children.keySet()) { - keys[i++] = key; + /** + * Finds the lowest base at which every label lands on a free slot. Labels + * arrive in surface-character order, not numeric order, so the smallest and + * largest label are computed rather than assumed positional. + */ + private int findBase(int[] labels, int labelCount) { + int smallest = labels[0]; + int largest = labels[0]; + for (int k = 1; k < labelCount; k++) { + smallest = Math.min(smallest, labels[k]); + largest = Math.max(largest, labels[k]); + } + int candidate = Math.max(1, watermark - smallest); + while (true) { + ensureCapacity(candidate + largest); + boolean fits = true; + for (int k = 0; fits && k < labelCount; k++) { + fits = check[candidate + labels[k]] == EMPTY; + } + if (fits) { + while (watermark < check.length && check[watermark] != EMPTY) { + watermark++; + } + return candidate; + } + candidate++; + } } - Arrays.sort(keys); - final TrieNode[] nodes = new TrieNode[keys.length]; - for (int k = 0; k < keys.length; k++) { - nodes[k] = children.get(keys[k]).freeze(); + + private void ensureCapacity(int slot) { + if (slot >= check.length) { + int capacity = check.length; + while (capacity <= slot) { + capacity += capacity >> 1; + } + base = Arrays.copyOf(base, capacity); + final int old = check.length; + check = Arrays.copyOf(check, capacity); + Arrays.fill(check, old, capacity, EMPTY); + } } - return new TrieNode(keys, nodes, entries); } } @@ -291,7 +487,7 @@ interface PrefixMatchConsumer { void accept(int length, List entries); } - private final TrieNode lexicon; + private final DoubleArrayLexicon lexicon; private final int maxSurfaceLength; private final short[] connectionCosts; private final int rightSize; @@ -300,7 +496,7 @@ interface PrefixMatchConsumer { private final Category defaultCategory; private final Map> unknownEntries; - private MecabDictionary(TrieNode lexicon, int maxSurfaceLength, + private MecabDictionary(DoubleArrayLexicon lexicon, int maxSurfaceLength, short[] connectionCosts, int rightSize, Map categories, CategoryTable categoryTable, Map> unknownEntries) { this.lexicon = lexicon; @@ -409,23 +605,8 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); - return new MecabDictionary(buildTrie(lexicon), maxSurface, costs, rightSize, - categories, categoryTable.build(categories), unknown); - } - - /** Folds the surface-keyed lexicon into a character trie for prefix search. */ - private static TrieNode buildTrie(Map> lexicon) { - final TrieBuilderNode root = new TrieBuilderNode(); - for (final Map.Entry> entry : lexicon.entrySet()) { - TrieBuilderNode node = root; - final String surface = entry.getKey(); - for (int i = 0; i < surface.length(); i++) { - node = node.children.computeIfAbsent(surface.charAt(i), - key -> new TrieBuilderNode()); - } - node.entries = List.copyOf(entry.getValue()); - } - return root.freeze(); + return new MecabDictionary(DoubleArrayLexicon.build(lexicon), maxSurface, costs, + rightSize, categories, categoryTable.build(categories), unknown); } /** @@ -534,11 +715,7 @@ private static void readCharacterDefinition(Path file, Charset charset, * @return The entries, or {@code null} when the surface is not listed. */ List lookup(String surface) { - TrieNode node = lexicon; - for (int i = 0; i < surface.length() && node != null; i++) { - node = node.child(surface.charAt(i)); - } - return node == null ? null : node.entries; + return lexicon.lookup(surface); } /** @@ -551,16 +728,7 @@ List lookup(String surface) { * @param consumer Receives each match. */ void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) { - TrieNode node = lexicon; - for (int i = from; i < to; i++) { - node = node.child(text.charAt(i)); - if (node == null) { - return; - } - if (node.entries != null) { - consumer.accept(i - from + 1, node.entries); - } - } + lexicon.prefixMatches(text, from, to, consumer); } /** @return The length in characters of the longest surface form in the lexicon. */ From cac8a31ea74493cde727d2c0978da6c82226bd20 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:58:38 -0400 Subject: [PATCH 10/24] OPENNLP-1894: Chain lattice nodes intrusively instead of allocating per-position lists The Viterbi lattice held one ArrayList per text position plus one fresh candidate list per position, pure allocation churn on long stretches. Nodes ending at a position now chain through their own link field behind a single head reference per position, and candidate gathering fills one scratch list reused across positions, so building the lattice allocates nothing besides the nodes themselves. IPADIC throughput on the 400k-character harness rises from 5.6M to 6.5M characters per second with identical output. --- .../tokenize/lattice/LatticeTokenizer.java | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index f7a479bf83..98f75d1b6b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -67,7 +67,11 @@ public LatticeTokenizer(MecabDictionary dictionary) { this.dictionary = dictionary; } - /** One lattice node: a candidate morpheme with its best path cost so far. */ + /** + * One lattice node: a candidate morpheme with its best path cost so far. Nodes + * ending at one position chain through {@link #nextEndingHere}, so the lattice + * needs one head reference per position instead of a list allocation. + */ private static final class Node { private final int start; private final int end; @@ -75,6 +79,7 @@ private static final class Node { private final boolean unknown; private long pathCost = Long.MAX_VALUE; private Node previous; + private Node nextEndingHere; private Node(int start, int end, WordEntry entry, boolean unknown) { this.start = start; @@ -136,12 +141,9 @@ public Span[] tokenizePos(String s) { /** Runs the Viterbi search over one whitespace-free stretch of text. */ private void decode(String text, int from, int to, List morphemes) { final int length = to - from; - final List> endingAt = new ArrayList<>(length + 1); - for (int i = 0; i <= length; i++) { - endingAt.add(new ArrayList<>()); - } - final boolean[] reachable = new boolean[length + 1]; - reachable[0] = true; + // One head reference per position; nodes ending there chain through the node's + // own link, so building the lattice allocates nothing besides the nodes. + final Node[] endingAt = new Node[length + 1]; // One right-to-left pass fixes each position's category and same-category run end, // so candidate generation reads them instead of rescanning the run from every @@ -150,23 +152,25 @@ private void decode(String text, int from, int to, List morphemes) { final int[] runEndAt = new int[length]; computeCategoryRuns(text, from, to, categoryAt, runEndAt); + final List candidates = new ArrayList<>(); for (int i = 0; i < length; i++) { - if (!reachable[i]) { + if (i > 0 && endingAt[i] == null) { continue; } - final List candidates = - candidates(text, from, to, i, categoryAt[i], runEndAt[i]); + candidates.clear(); + candidates(text, from, to, i, categoryAt[i], runEndAt[i], candidates); for (final Node candidate : candidates) { - relax(candidate, i == 0 ? null : endingAt.get(i)); + relax(candidate, i == 0 ? null : endingAt[i]); if (candidate.pathCost < Long.MAX_VALUE) { - endingAt.get(candidate.end - from).add(candidate); - reachable[candidate.end - from] = true; + final int end = candidate.end - from; + candidate.nextEndingHere = endingAt[end]; + endingAt[end] = candidate; } } } Node best = null; - for (final Node node : endingAt.get(length)) { + for (Node node = endingAt[length]; node != null; node = node.nextEndingHere) { final long total = node.pathCost + dictionary.connectionCost(node.entry.rightId(), BOUNDARY_CONTEXT); if (best == null || total < best.pathCost @@ -190,13 +194,14 @@ private void decode(String text, int from, int to, List morphemes) { } /** Connects a candidate to the cheapest predecessor ending where it starts. */ - private void relax(Node candidate, List predecessors) { + private void relax(Node candidate, Node predecessors) { if (predecessors == null) { candidate.pathCost = candidate.entry.cost() + dictionary.connectionCost(BOUNDARY_CONTEXT, candidate.entry.leftId()); return; } - for (final Node predecessor : predecessors) { + for (Node predecessor = predecessors; predecessor != null; + predecessor = predecessor.nextEndingHere) { final long total = predecessor.pathCost + dictionary.connectionCost(predecessor.entry.rightId(), candidate.entry.leftId()) + candidate.entry.cost(); @@ -238,10 +243,9 @@ private void computeCategoryRuns(String text, int from, int to, } /** Gathers lexicon matches and unknown-word candidates starting at one position. */ - private List candidates(String text, int from, int to, int offset, - Category positionCategory, int positionRunEnd) { + private void candidates(String text, int from, int to, int offset, + Category positionCategory, int positionRunEnd, List candidates) { final int position = from + offset; - final List candidates = new ArrayList<>(); final boolean[] matched = new boolean[1]; dictionary.prefixMatches(text, position, to, (length, entries) -> { matched[0] = true; @@ -285,7 +289,6 @@ private List candidates(String text, int from, int to, int offset, throw new IllegalStateException("dictionary provides no candidate at position " + position + "; unk.def lacks a DEFAULT template"); } - return candidates; } /** From f28c02fc654cfec2422179b1d0eeb310e953dbff Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:44:21 -0400 Subject: [PATCH 11/24] OPENNLP-1894: Document lattice CJK tokenization with a mirror-tested example Add a lattice tokenizer section to the manual citing LatticeUsageExampleTest. --- opennlp-docs/src/docbkx/tokenizer.xml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index cd1d8a2ddf..fbc74fbb2e 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -538,5 +538,30 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { +
+ Lattice tokenization for CJK + + Languages written without spaces need a dictionary-backed segmenter. + LatticeTokenizer scores paths over a mecab-format dictionary and + emits the cheapest segmentation with spans in original text coordinates. + UnigramSegmenter does the same from a plain frequency lexicon. + Install a dictionary archive with MecabDictionaryInstaller, load it + as a MecabDictionary, and tokenize. + LatticeUsageExampleTest asserts the install-load-tokenize and + lexicon flows shown here. + + morphemes = tokenizer.analyze(text); + +UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); +String[] words = segmenter.tokenize(text);]]> + + +
From 701f1db5709c80d2a02bd7bd4f2e8cc5ba11993e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 06:39:29 -0400 Subject: [PATCH 12/24] OPENNLP-1894: Apply the review-convention pass and drop unreferenced lexicon accessors --- .../dev => dev}/README-mecab-dictionaries.md | 10 +- .../dev => dev}/download-mecab-dictionary.sh | 4 +- .../tokenize/lattice/LatticeTokenizer.java | 13 +-- .../tokenize/lattice/MecabDictionary.java | 93 ++++-------------- .../lattice/MecabDictionaryInstaller.java | 22 +++-- .../tokenize/lattice/UnigramSegmenter.java | 19 ++-- .../lattice/LatticeTokenizerTest.java | 6 +- .../lattice/LatticeUsageExampleTest.java | 68 +------------ .../lattice/MecabDictionaryInstallerTest.java | 55 ++--------- .../tools/tokenize/lattice/TarGzArchives.java | 97 +++++++++++++++++++ .../lattice/UnigramSegmenterTest.java | 8 +- opennlp-docs/src/docbkx/tokenizer.xml | 2 +- 12 files changed, 171 insertions(+), 226 deletions(-) rename {opennlp-core/opennlp-runtime/dev => dev}/README-mecab-dictionaries.md (88%) rename {opennlp-core/opennlp-runtime/dev => dev}/download-mecab-dictionary.sh (94%) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java diff --git a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md similarity index 88% rename from opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md rename to dev/README-mecab-dictionaries.md index c3244ba9c4..fce653b397 100644 --- a/opennlp-core/opennlp-runtime/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -17,7 +17,7 @@ # CJK dictionaries for the lattice tokenizer -The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and by using it you accept that dictionary's license. Read the license file inside the archive before use; nothing in this repository grants you those terms. +The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and each dictionary carries its own license. Read the license file inside the archive before use. ## Known mecab-format dictionary projects @@ -63,7 +63,8 @@ import opennlp.tools.tokenize.lattice.MecabDictionary; MecabDictionary dictionary = MecabDictionary.load(Path.of("ipadic"), Charset.forName("EUC-JP")); LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary); -String[] tokens = tokenizer.tokenize("東京都に行く"); +// "Tokyo-to ni iku" (go to the Tokyo metropolis), escaped to keep this file ASCII +String[] tokens = tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); ``` For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` is enough. Loaded dictionaries and tokenizers are immutable and safe to share between threads, so load once and reuse. @@ -77,7 +78,8 @@ import java.nio.file.Path; import opennlp.tools.tokenize.lattice.UnigramSegmenter; UnigramSegmenter segmenter = UnigramSegmenter.load(Path.of("words.txt")); -String[] tokens = segmenter.tokenize("我来到北京天安门"); +// "wo laidao Beijing Tian'anmen" (I arrive at Beijing Tiananmen), escaped as above +String[] tokens = segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u5929\u5B89\u95E8"); ``` -As with the dictionaries, the lexicon's license is between you and its publisher; nothing is bundled. +As with the dictionaries, the lexicon carries its own license; nothing is bundled. diff --git a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh b/dev/download-mecab-dictionary.sh similarity index 94% rename from opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh rename to dev/download-mecab-dictionary.sh index 30c5b23bde..3b86ed09d6 100755 --- a/opennlp-core/opennlp-runtime/dev/download-mecab-dictionary.sh +++ b/dev/download-mecab-dictionary.sh @@ -23,8 +23,8 @@ # this directory for the known dictionary projects, their encodings, and the Java # steps that follow the download. # -# The dictionary you download carries its own license, which you accept by using it. -# Nothing is bundled with Apache OpenNLP and no dictionary location is built in. +# The dictionary you download carries its own license. Nothing is bundled with +# Apache OpenNLP and no dictionary location is built in. set -euo pipefail diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 98f75d1b6b..5ee2384ba1 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -69,8 +69,7 @@ public LatticeTokenizer(MecabDictionary dictionary) { /** * One lattice node: a candidate morpheme with its best path cost so far. Nodes - * ending at one position chain through {@link #nextEndingHere}, so the lattice - * needs one head reference per position instead of a list allocation. + * ending at one position chain through {@link #nextEndingHere}. */ private static final class Node { private final int start; @@ -141,13 +140,9 @@ public Span[] tokenizePos(String s) { /** Runs the Viterbi search over one whitespace-free stretch of text. */ private void decode(String text, int from, int to, List morphemes) { final int length = to - from; - // One head reference per position; nodes ending there chain through the node's - // own link, so building the lattice allocates nothing besides the nodes. + // Each element heads the chain of nodes ending at that position. final Node[] endingAt = new Node[length + 1]; - // One right-to-left pass fixes each position's category and same-category run end, - // so candidate generation reads them instead of rescanning the run from every - // position, which would cost the square of the run length on long uniform runs. final Category[] categoryAt = new Category[length]; final int[] runEndAt = new int[length]; computeCategoryRuns(text, from, to, categoryAt, runEndAt); @@ -259,9 +254,9 @@ private void candidates(String text, int from, int to, int offset, final Category category; final int runEnd; if (positionCategory == null) { - // Only a lexicon surface ending inside a surrogate pair could make such a + // Only a lexicon surface ending inside a surrogate pair can make such a // position reachable; classify the stray code unit on the spot so the lattice - // stays connected the way it always did. + // stays connected. category = dictionary.categoryOf(codePoint); runEnd = position + Character.charCount(codePoint); } else { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index cd1f46ba7f..ed0775a19e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -35,9 +35,8 @@ * An immutable, in-memory dictionary in the mecab directory format: lexicon entries * from the {@code *.csv} files, connection costs from {@code matrix.def}, character * categories from {@code char.def}, and unknown-word templates from {@code unk.def}, - * loaded from a user-supplied dictionary directory. The reader is a clean-room - * implementation of the documented format; no dictionary data is bundled or downloaded - * by this class, so the dictionaries' own licenses never attach to this library. + * loaded from a user-supplied dictionary directory. No dictionary data is bundled or + * downloaded by this class. * *

The same format serves multiple languages: the Japanese IPADIC and UniDic * distributions and the Korean mecab-ko-dic all load through this one reader, with the @@ -60,12 +59,9 @@ record Category(String name, boolean invoke, boolean group, int length) { /** * The lexicon as a double-array trie: one transition is one array read and one - * comparison, so the per-position prefix walk of the tokenizer never hashes, never - * binary-searches a fan-out, and never boxes. Characters are recoded into dense - * labels ordered by frequency before the array is built, which keeps the array - * compact although CJK surfaces draw from tens of thousands of distinct - * characters; a character the lexicon never uses misses in the recode table before - * the array is even consulted. + * comparison. Characters are recoded into dense labels ordered by descending + * frequency before the array is built, which keeps the array compact; a character + * the lexicon never uses misses in the recode table before the array is consulted. * *

The layout is the classic base/check pair: from state {@code s}, label * {@code c} leads to {@code t = base[s] + c} exactly when {@code check[t] == s}. @@ -160,32 +156,6 @@ private void prefixMatches(String text, int from, int to, } } - /** - * Looks up the entries of an exact surface form. - * - * @param surface The surface form. - * @return The entries, or {@code null} when the surface is not listed. - */ - private List lookup(String surface) { - int state = Builder.ROOT; - for (int i = 0; i < surface.length(); i++) { - final int code = codeOf[surface.charAt(i)]; - if (code < 0) { - return null; - } - final int next = base[state] + code; - if (next >= check.length || check[next] != state) { - return null; - } - state = next; - } - final int terminal = base[state]; - if (terminal < check.length && check[terminal] == state && base[terminal] < 0) { - return values[-base[terminal] - 1]; - } - return null; - } - /** * The recursive sorted-range builder: each call places one node's children by * finding a base at which every child label lands on a free slot, then recurses @@ -310,12 +280,9 @@ private void ensureCapacity(int slot) { * The {@code char.def} code point to category name mapping, over the whole Unicode * code point range. * - *

The Basic Multilingual Plane is held in a directly indexed array, which is one - * reference per BMP code point and is the entire mapping for a BMP-only dictionary. - * The supplementary planes are held as a sorted, non-overlapping range table searched - * by binary search, because dictionaries map them in a handful of large blocks: a - * directly indexed array over all of Unicode would spend more than four megabytes of - * references to say what a few range rows already say.

+ *

The Basic Multilingual Plane is held in a directly indexed array. The + * supplementary planes are held as a sorted, non-overlapping range table searched by + * binary search, because dictionaries map them in a handful of large blocks.

*/ private static final class CategoryTable { @@ -334,10 +301,8 @@ private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, /** * Looks up the category a {@code char.def} mapping gives a code point. The table - * holds the {@link Category} instances themselves, so a lookup on the tokenizer's - * per-character path costs one array read or one binary search, never a name map - * access, and two code points of one category share one instance to compare by - * identity. + * holds the {@link Category} instances themselves, and two code points of one + * category share one instance, so categories may be compared by identity. * * @param codePoint The code point to classify. * @return The category, or {@code null} when no mapping covers the code point. @@ -488,7 +453,6 @@ interface PrefixMatchConsumer { } private final DoubleArrayLexicon lexicon; - private final int maxSurfaceLength; private final short[] connectionCosts; private final int rightSize; private final Map categories; @@ -496,11 +460,10 @@ interface PrefixMatchConsumer { private final Category defaultCategory; private final Map> unknownEntries; - private MecabDictionary(DoubleArrayLexicon lexicon, int maxSurfaceLength, + private MecabDictionary(DoubleArrayLexicon lexicon, short[] connectionCosts, int rightSize, Map categories, CategoryTable categoryTable, Map> unknownEntries) { this.lexicon = lexicon; - this.maxSurfaceLength = maxSurfaceLength; this.connectionCosts = connectionCosts; this.rightSize = rightSize; this.categories = categories; @@ -536,8 +499,11 @@ public static MecabDictionary load(Path directory) throws IOException { * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static MecabDictionary load(Path directory, Charset charset) throws IOException { - if (directory == null || charset == null) { - throw new IllegalArgumentException("directory and charset must not be null"); + if (directory == null) { + throw new IllegalArgumentException("directory must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } // The connection matrix is read first because its dimensions are what every // lexicon entry's context ids have to be inside of. @@ -587,11 +553,9 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final Map> lexicon = new HashMap<>(); - int maxSurface = 0; try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { for (final Path csv : csvFiles) { - maxSurface = Math.max(maxSurface, - readLexicon(csv, charset, lexicon, leftSize, rightSize)); + readLexicon(csv, charset, lexicon, leftSize, rightSize); } } if (lexicon.isEmpty()) { @@ -605,7 +569,7 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map> unknown = new HashMap<>(); readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); - return new MecabDictionary(DoubleArrayLexicon.build(lexicon), maxSurface, costs, + return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, rightSize, categories, categoryTable.build(categories), unknown); } @@ -620,14 +584,12 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc * ids. * @param rightSize The second {@code matrix.def} dimension, which bounds left context * ids. - * @return The length in characters of the longest surface form read. * @throws IOException Thrown if the file is missing, an entry is malformed, or an * entry's context id is outside the matrix dimensions. */ - private static int readLexicon(Path file, Charset charset, + private static void readLexicon(Path file, Charset charset, Map> target, int leftSize, int rightSize) throws IOException { - int maxSurface = 0; int lineNumber = 0; for (final String line : readLines(file, charset)) { lineNumber++; @@ -658,9 +620,7 @@ private static int readLexicon(Path file, Charset charset, parseInt(fields.get(3), file.toString(), lineNumber), List.copyOf(fields.subList(4, fields.size()))); target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); - maxSurface = Math.max(maxSurface, surface.length()); } - return maxSurface; } /** Reads char.def: category behavior lines and code point mapping lines. */ @@ -708,16 +668,6 @@ private static void readCharacterDefinition(Path file, Charset charset, } } - /** - * Looks up the lexicon entries for an exact surface form. - * - * @param surface The surface form. - * @return The entries, or {@code null} when the surface is not listed. - */ - List lookup(String surface) { - return lexicon.lookup(surface); - } - /** * Reports every lexicon surface starting at a text position, walking the trie once * with no substring allocation. @@ -731,11 +681,6 @@ void prefixMatches(String text, int from, int to, PrefixMatchConsumer consumer) lexicon.prefixMatches(text, from, to, consumer); } - /** @return The length in characters of the longest surface form in the lexicon. */ - int maxSurfaceLength() { - return maxSurfaceLength; - } - /** * Reads the connection cost between two adjacent nodes. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 9bbbde7902..3c2589bc02 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -29,9 +29,8 @@ /** * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the * dictionary is acquired by the user at install time and never ships with this library. - * The user supplies the archive location from the dictionary project of their choice - * and thereby accepts that dictionary's license; nothing is bundled and no location is - * built in. + * The archive location is user-supplied; no dictionary data is bundled and no download + * location is built in. * *

The installer reads gzip-compressed tar archives, the format the common * distributions use, and extracts only the dictionary payload: the {@code *.csv} @@ -64,11 +63,15 @@ private MecabDictionaryInstaller() { * @return The number of dictionary files extracted. * @throws IOException Thrown if fetching, reading, or writing fails, or the archive * contains no dictionary file. - * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + * @throws IllegalArgumentException Thrown if a parameter is {@code null} or + * {@code archive} is not an absolute URI. */ public static int install(URI archive, Path targetDirectory) throws IOException { - if (archive == null || targetDirectory == null) { - throw new IllegalArgumentException("archive and targetDirectory must not be null"); + if (archive == null) { + throw new IllegalArgumentException("archive must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); } try (InputStream in = archive.toURL().openStream()) { return extract(in, targetDirectory); @@ -89,8 +92,11 @@ public static int install(URI archive, Path targetDirectory) throws IOException */ public static int extract(InputStream archiveStream, Path targetDirectory) throws IOException { - if (archiveStream == null || targetDirectory == null) { - throw new IllegalArgumentException("stream and targetDirectory must not be null"); + if (archiveStream == null) { + throw new IllegalArgumentException("archiveStream must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); } Files.createDirectories(targetDirectory); final InputStream tar = new GZIPInputStream(archiveStream); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index b4cfaa8ea6..ba3e06861f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -41,9 +41,8 @@ * and counts. * *

The lexicon format is one entry per line: the word, its count, and optionally a - * tag, separated by whitespace. The user supplies the lexicon file and thereby accepts - * its license; nothing is bundled. Every reported span is in original text - * coordinates.

+ * tag, separated by whitespace. The lexicon file is user-supplied; no lexicon data is + * bundled. Every reported span is in original text coordinates.

* *

Instances are immutable and safe to share between threads.

* @@ -90,8 +89,11 @@ public static UnigramSegmenter load(Path lexicon) throws IOException { * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOException { - if (lexicon == null || charset == null) { - throw new IllegalArgumentException("lexicon and charset must not be null"); + if (lexicon == null) { + throw new IllegalArgumentException("lexicon must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } try (InputStream in = Files.newInputStream(lexicon)) { return load(in, charset); @@ -109,8 +111,11 @@ public static UnigramSegmenter load(Path lexicon, Charset charset) throws IOExce */ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) throws IOException { - if (lexiconStream == null || charset == null) { - throw new IllegalArgumentException("stream and charset must not be null"); + if (lexiconStream == null) { + throw new IllegalArgumentException("lexiconStream must not be null"); + } + if (charset == null) { + throw new IllegalArgumentException("charset must not be null"); } final Map counts = new HashMap<>(); long total = 0; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index e1be5a4a98..034cd6d8c3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -99,7 +99,7 @@ void testMorphemesCarryDictionaryFeatures() { Assertions.assertEquals(4, morphemes.size()); Assertions.assertEquals(List.of("noun", "proper"), morphemes.get(0).features()); Assertions.assertEquals(List.of("particle", "case"), morphemes.get(2).features()); - Assertions.assertEquals(false, morphemes.get(0).unknown()); + Assertions.assertFalse(morphemes.get(0).unknown()); } @Test @@ -107,7 +107,7 @@ void testUnknownLatinRunGroupsIntoOneMorpheme() { final List morphemes = tokenizer.analyze("ABC\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("ABC", morphemes.get(0).surface()); - Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(0).unknown()); Assertions.assertEquals(List.of("noun", "foreign"), morphemes.get(0).features()); } @@ -116,7 +116,7 @@ void testUnknownKanjiPreferOneMorphemeOverTwo() { final List morphemes = tokenizer.analyze("\u5CE0\u9053\u306B\u884C\u304F"); Assertions.assertEquals(3, morphemes.size()); Assertions.assertEquals("\u5CE0\u9053", morphemes.get(0).surface()); - Assertions.assertEquals(true, morphemes.get(0).unknown()); + Assertions.assertTrue(morphemes.get(0).unknown()); } /** diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index a61969c902..0fc58585de 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -17,14 +17,12 @@ package opennlp.tools.tokenize.lattice; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.zip.GZIPOutputStream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -50,70 +48,6 @@ */ public class LatticeUsageExampleTest { - /** - * Appends one ustar file entry to a growing tar image: a 512-byte header block - * followed by the content padded to a block boundary. - * - * @param tar The tar image under construction. Must not be {@code null}. - * @param name The entry name including any directory prefix. Must not be - * {@code null}, empty, or longer than the 100-byte header name field. - * @param content The entry content bytes. Must not be {@code null}. - * @throws IOException Thrown if writing to the in-memory stream fails. - * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. - */ - private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) - throws IOException { - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - if (nameBytes.length == 0 || nameBytes.length > 100) { - throw new IllegalArgumentException( - "entry name must be 1..100 bytes, got " + nameBytes.length); - } - final byte[] header = new byte[512]; - System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); - final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { - header[i] = ' '; - } - int checksum = 0; - for (final byte b : header) { - checksum += b & 0xFF; - } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; - tar.write(header); - tar.write(content); - final int padding = (512 - content.length % 512) % 512; - tar.write(new byte[padding]); - } - - /** - * Builds a gzip-compressed tar archive from name and content pairs, the layout a - * dictionary distribution ships in. - * - * @param entries The entries as {@code {name, content}} pairs. Must not be - * {@code null}. - * @return The compressed archive bytes. Never {@code null}. - * @throws IOException Thrown if writing to the in-memory streams fails. - */ - private static byte[] gzippedTar(String[][] entries) throws IOException { - final ByteArrayOutputStream tar = new ByteArrayOutputStream(); - for (final String[] entry : entries) { - tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); - } - tar.write(new byte[1024]); - final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); - try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { - gzip.write(tar.toByteArray()); - } - return compressed.toByteArray(); - } - /** * Walks the full mecab-format flow: package a miniature Japanese dictionary as a * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The @@ -126,7 +60,7 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) throws IOException { // A minimal but complete dictionary: one lexicon file plus the three definition // files every mecab-format distribution contains, wrapped like a release archive. - final byte[] archive = gzippedTar(new String[][] { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { {"mini-dict-0.1/lexicon.csv", String.join("\n", "\u6771\u4EAC,0,0,3000,noun,proper", "\u4EAC\u90FD,0,0,3000,noun,proper", diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index fe999e4609..da95c97366 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -18,64 +18,24 @@ package opennlp.tools.tokenize.lattice; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.zip.GZIPOutputStream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +/** + * Tests the installer against project-authored, in-memory archives; no external + * dictionary data and no network access are involved. + */ public class MecabDictionaryInstallerTest { - /** Writes one ustar entry: a 512-byte header block and padded content. */ - private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) - throws IOException { - final byte[] header = new byte[512]; - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); - final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { - header[i] = ' '; - } - int checksum = 0; - for (final byte b : header) { - checksum += b & 0xFF; - } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; - tar.write(header); - tar.write(content); - final int padding = (512 - content.length % 512) % 512; - tar.write(new byte[padding]); - } - - private static byte[] archive(String[][] entries) throws IOException { - final ByteArrayOutputStream tar = new ByteArrayOutputStream(); - for (final String[] entry : entries) { - tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); - } - tar.write(new byte[1024]); - final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); - try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { - gzip.write(tar.toByteArray()); - } - return compressed.toByteArray(); - } - @Test void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) throws IOException { - final byte[] archive = archive(new String[][] { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { {"dict-1.0/lexicon.csv", "cat,0,0,100,noun\n"}, {"dict-1.0/matrix.def", "1 1\n0 0 0\n"}, {"dict-1.0/char.def", "DEFAULT 0 1 0\n"}, @@ -101,7 +61,7 @@ void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) throws IOException { final Path archiveFile = source.resolve("dict.tar.gz"); - Files.write(archiveFile, archive(new String[][] { + Files.write(archiveFile, TarGzArchives.gzippedTar(new String[][] { {"d/words.csv", "cat,0,0,100,noun\n"}, {"d/matrix.def", "1 1\n0 0 0\n"}})); @@ -115,7 +75,8 @@ void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) @Test void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) throws IOException { - final byte[] archive = archive(new String[][] {{"readme.txt", "nothing here"}}); + final byte[] archive = + TarGzArchives.gzippedTar(new String[][] {{"readme.txt", "nothing here"}}); Assertions.assertThrows(IOException.class, () -> MecabDictionaryInstaller.extract( new ByteArrayInputStream(archive), target)); } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java new file mode 100644 index 0000000000..4fdfb29d01 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPOutputStream; + +/** + * Builds miniature, project-authored gzip-compressed ustar archives in memory for the + * tests of this package; no external archive data is involved. + */ +final class TarGzArchives { + + private TarGzArchives() { + } + + /** + * Builds a gzip-compressed tar archive from name and content pairs, the layout a + * dictionary distribution ships in. + * + * @param entries The entries as {@code {name, content}} pairs. Must not be + * {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + static byte[] gzippedTar(String[][] entries) throws IOException { + final ByteArrayOutputStream tar = new ByteArrayOutputStream(); + for (final String[] entry : entries) { + tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + } + tar.write(new byte[1024]); + final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { + gzip.write(tar.toByteArray()); + } + return compressed.toByteArray(); + } + + /** + * Appends one ustar file entry to a growing tar image: a 512-byte header block + * followed by the content padded to a block boundary. + * + * @param tar The tar image under construction. Must not be {@code null}. + * @param name The entry name including any directory prefix. Must not be + * {@code null}, empty, or longer than the 100-byte header name field. + * @param content The entry content bytes. Must not be {@code null}. + * @throws IOException Thrown if writing to the in-memory stream fails. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + */ + private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) + throws IOException { + final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + if (nameBytes.length == 0 || nameBytes.length > 100) { + throw new IllegalArgumentException( + "entry name must be 1..100 bytes, got " + nameBytes.length); + } + final byte[] header = new byte[512]; + System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); + final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); + System.arraycopy(mode, 0, header, 100, mode.length); + final String size = String.format("%011o", content.length); + System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); + header[156] = '0'; + for (int i = 148; i < 156; i++) { + header[i] = ' '; + } + int checksum = 0; + for (final byte b : header) { + checksum += b & 0xFF; + } + final String checksumText = String.format("%06o", checksum); + System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); + header[154] = 0; + header[155] = ' '; + tar.write(header); + tar.write(content); + final int padding = (512 - content.length % 512) % 512; + tar.write(new byte[padding]); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index e6ebd604e1..f5cff587bc 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -183,19 +183,19 @@ void testInvalidArguments() { * span over both of its surrogate halves, and no span boundary ever lands between * them. */ - @org.junit.jupiter.api.Test + @Test void testUnknownSupplementaryCharacterIsNeverSplit() { // U+20BB7, a CJK extension B ideograph, written as its surrogate pair final String text = "\uD842\uDFB7\uD842\uDFB7"; - final opennlp.tools.util.Span[] spans = segmenter.tokenizePos(text); - for (final opennlp.tools.util.Span span : spans) { + final Span[] spans = segmenter.tokenizePos(text); + for (final Span span : spans) { Assertions.assertEquals(0, span.getStart() % 2, "span must start on a code point boundary: " + span); Assertions.assertEquals(0, span.getEnd() % 2, "span must end on a code point boundary: " + span); } int covered = 0; - for (final opennlp.tools.util.Span span : spans) { + for (final Span span : spans) { covered += span.length(); } Assertions.assertEquals(text.length(), covered); diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index fbc74fbb2e..6f74b6a601 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -542,7 +542,7 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { Lattice tokenization for CJK Languages written without spaces need a dictionary-backed segmenter. - LatticeTokenizer scores paths over a mecab-format dictionary and + LatticeTokenizer scores paths over a MeCab-format dictionary and emits the cheapest segmentation with spans in original text coordinates. UnigramSegmenter does the same from a plain frequency lexicon. Install a dictionary archive with MecabDictionaryInstaller, load it From fdc9f93a4da74a5ef464ee56d7ffb1b96c67714f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 24 Jul 2026 14:53:05 -0400 Subject: [PATCH 13/24] OPENNLP-1894: Trim parsed lines as Unicode whitespace and document the tokenizer overrides The frequency lexicon was trimmed with String.trim(), which strips only ASCII control characters and the space. A line starting with an ideographic space (U+3000), ordinary in hand-edited CJK text files, therefore kept that space as part of the word and pushed the count field one token to the right, so the load failed as a malformed count. The lexicon reader now trims with StringUtil.trimUnicodeWhitespace, matching the White_Space convention the rest of the tokenizer already scans by, and a test pins the leading U+3000 case. The mecab reader's line and numeric-field trims move to the same call so one class does not mix two whitespace judgments; those fields are ASCII in valid dictionaries, so the behavior there is unchanged. Both tokenizer views also gain {@inheritDoc} and their null contract, and the unknown-candidate helper drops a static modifier it did not need. --- .../tools/tokenize/lattice/LatticeTokenizer.java | 16 +++++++++++++++- .../tools/tokenize/lattice/MecabDictionary.java | 8 ++++---- .../tools/tokenize/lattice/UnigramSegmenter.java | 16 +++++++++++++++- .../tokenize/lattice/UnigramSegmenterTest.java | 15 +++++++++++++++ 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 5ee2384ba1..7ce8be2aa4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -117,6 +117,13 @@ public List analyze(String text) { return morphemes; } + /** + * {@inheritDoc} + * + *

Reports the segmented surfaces, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public String[] tokenize(String s) { final List morphemes = analyze(s); @@ -127,6 +134,13 @@ public String[] tokenize(String s) { return tokens; } + /** + * {@inheritDoc} + * + *

Reports the segmented spans in original text coordinates, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public Span[] tokenizePos(String s) { final List morphemes = analyze(s); @@ -301,7 +315,7 @@ private void candidates(String text, int from, int to, int offset, * @param category The category of that run. * @param templates The category's unknown-word templates. */ - private static void addUnknown(List candidates, String text, int position, + private void addUnknown(List candidates, String text, int position, int runEnd, Category category, List templates) { if (category.group()) { for (final WordEntry entry : templates) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index ed0775a19e..a971c586a3 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -528,7 +528,7 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final short[] costs = new short[(int) cells]; for (int i = 1; i < matrixLines.size(); i++) { - final String line = matrixLines.get(i).trim(); + final String line = StringUtil.trimUnicodeWhitespace(matrixLines.get(i)); if (line.isEmpty()) { continue; } @@ -630,7 +630,7 @@ private static void readCharacterDefinition(Path file, Charset charset, int lineNumber = 0; for (final String raw : readLines(file, charset)) { lineNumber++; - final String line = stripComment(raw).trim(); + final String line = StringUtil.trimUnicodeWhitespace(stripComment(raw)); if (line.isEmpty()) { continue; } @@ -809,7 +809,7 @@ private static String[] splitWhitespace(String line) { private static int parseInt(String text, String file, int lineNumber) throws IOException { try { - return Integer.parseInt(text.trim()); + return Integer.parseInt(StringUtil.trimUnicodeWhitespace(text)); } catch (NumberFormatException e) { throw new IOException("malformed number in " + file + " line " + lineNumber, e); } @@ -829,7 +829,7 @@ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { final int codePoint; try { - codePoint = Integer.parseInt(text.trim().substring(2), 16); + codePoint = Integer.parseInt(StringUtil.trimUnicodeWhitespace(text).substring(2), 16); } catch (RuntimeException e) { throw new IOException("malformed code point in " + file + " line " + lineNumber, e); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index ba3e06861f..00469ac4f7 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -127,7 +127,7 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) continue; } lineNumber++; - final String line = content.substring(lineStart, i).trim(); + final String line = StringUtil.trimUnicodeWhitespace(content.substring(lineStart, i)); lineStart = i + 1; if (line.isEmpty()) { continue; @@ -178,6 +178,13 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) return new UnigramSegmenter(root, unknown); } + /** + * {@inheritDoc} + * + *

Reports the segmented surfaces, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public String[] tokenize(String s) { final Span[] spans = tokenizePos(s); @@ -188,6 +195,13 @@ public String[] tokenize(String s) { return tokens; } + /** + * {@inheritDoc} + * + *

Reports the segmented spans in original text coordinates, whitespace omitted.

+ * + * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + */ @Override public Span[] tokenizePos(String s) { if (s == null) { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index f5cff587bc..977d42fcc4 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -200,4 +200,19 @@ void testUnknownSupplementaryCharacterIsNeverSplit() { } Assertions.assertEquals(text.length(), covered); } + + /** + * Pins Unicode-whitespace trimming of lexicon lines: a leading ideographic space + * (U+3000), common in hand-edited CJK text files, is stripped like ASCII whitespace, + * so the entry loads rather than failing as a malformed count. + */ + @Test + void testLeadingIdeographicSpaceIsTrimmed() throws IOException { + // U+3000 ideographic space, then the fixture word U+6211 and its count + final String lexicon = "\u3000\u6211 5000 r\n"; + final UnigramSegmenter loaded = UnigramSegmenter.load( + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + Assertions.assertArrayEquals(new String[] {"\u6211"}, loaded.tokenize("\u6211")); + } } From 7a98076c81b11478cd7f5041072bab32508e90cc Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:01:00 -0400 Subject: [PATCH 14/24] OPENNLP-1894: Address review: complete javadoc, hoist constants, and fold fixture duplication - Document the private lattice helpers decode, relax, and candidates, and the installer's boundedStream, with the parameter, return, and exception contracts the review expects every method to carry. - Document the WordEntry and Category record components and the double-array builder's findBase and ensureCapacity helpers. - Record on analyze, tokenize, and tokenizePos that a unk.def without a DEFAULT template leaves the lattice disconnected and makes them throw IllegalStateException. - State on readLines that it never returns an empty list, which is what lets the matrix.def header be read before the emptiness check. - Rename the Tokenizer override parameter from s to text in LatticeTokenizer and UnigramSegmenter, so the javadoc names a parameter that exists. - Hoist the matrix.def, char.def, and unk.def file names, the DEFAULT category name, the 0x code point prefix, the .. range separator, and the flag value into named constants in MecabDictionary, and let LatticeTokenizer reach the DEFAULT name through MecabDictionary instead of repeating the literal. - Name the tar block size, header field offsets, and field lengths in the TarGzArchives test helper instead of writing 512, 124, and 148 inline. - Replace the boolean[1] capture in candidates with a check that the candidate list is still empty, which is the same signal without the array. - Track the best boundary total in decode instead of recomputing the incumbent's connection cost on every comparison. - Drop the categories map field from MecabDictionary, which nothing read once the constructor resolved the DEFAULT category out of it. - Match the char.def code point prefix once, case insensitively, rather than testing 0x and 0X separately, and cut the range at the separator's own length. - Trim the matrix.def header before parsing it and report an empty first line as an empty matrix.def, since readLines never yields the empty list the previous check was looking for. - Split the omnibus malformed-dictionary test into named cases that pin the messages for a missing definition file, a char.def without DEFAULT, a lexicon with no entries, and an empty matrix.def. - Parameterize the malformed char.def cases and the malformed unigram lexicon cases, which were repeated assertThrows calls over one fixture shape. - Add a Morpheme test pinning the null and empty argument rejections and the defensive copy of the feature list. - Extend the invalid-argument tests to the entry points that were uncovered: MecabDictionary.load with a null directory or charset, the installer's null target, UnigramSegmenter's path and stream overloads, and both tokenize methods of each tokenizer. - Fold the repeated Files.write fixture calls into one write helper and hoist the shared lexicon, matrix, char.def, and unk.def fixture text into constants. - Correct dev/README-mecab-dictionaries.md to say that dicrc is the configuration file the distributions ship alongside the csv and def files a MecabDictionary reads, rather than implying the dictionary reads dicrc itself. --- dev/README-mecab-dictionaries.md | 2 +- .../tokenize/lattice/LatticeTokenizer.java | 65 ++++- .../tokenize/lattice/MecabDictionary.java | 140 ++++++--- .../lattice/MecabDictionaryInstaller.java | 10 +- .../tokenize/lattice/UnigramSegmenter.java | 32 +- .../lattice/LatticeTokenizerTest.java | 276 ++++++++++++------ .../lattice/MecabDictionaryInstallerTest.java | 2 + .../tools/tokenize/lattice/TarGzArchives.java | 48 ++- .../lattice/UnigramSegmenterTest.java | 32 +- 9 files changed, 424 insertions(+), 183 deletions(-) diff --git a/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md index fce653b397..6d1be9420f 100644 --- a/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -40,7 +40,7 @@ On the first run without a checksum the script prints the SHA-256 it computed; r ## Step 2: unpack with the installer -The installer extracts only the files a `MecabDictionary` reads (`*.csv`, `*.def`, and `dicrc`), flattens them into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: +The installer extracts only the dictionary payload: the `*.csv` and `*.def` files a `MecabDictionary` reads, plus the `dicrc` configuration file the distributions ship alongside them. It flattens the entries into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: ```java import java.nio.file.Path; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java index 7ce8be2aa4..649be0b80d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java @@ -95,6 +95,8 @@ private Node(int start, int end, WordEntry entry, boolean unknown) { * @return The morphemes in text order, spans in original coordinates, whitespace * omitted. Never {@code null}; empty for empty or all-whitespace input. * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position, which a {@code unk.def} without a {@code DEFAULT} template does. */ public List analyze(String text) { if (text == null) { @@ -122,11 +124,13 @@ public List analyze(String text) { * *

Reports the segmented surfaces, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. */ @Override - public String[] tokenize(String s) { - final List morphemes = analyze(s); + public String[] tokenize(String text) { + final List morphemes = analyze(text); final String[] tokens = new String[morphemes.size()]; for (int i = 0; i < tokens.length; i++) { tokens[i] = morphemes.get(i).surface(); @@ -139,11 +143,13 @@ public String[] tokenize(String s) { * *

Reports the segmented spans in original text coordinates, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. */ @Override - public Span[] tokenizePos(String s) { - final List morphemes = analyze(s); + public Span[] tokenizePos(String text) { + final List morphemes = analyze(text); final Span[] spans = new Span[morphemes.size()]; for (int i = 0; i < spans.length; i++) { spans[i] = morphemes.get(i).span(); @@ -151,7 +157,15 @@ public Span[] tokenizePos(String s) { return spans; } - /** Runs the Viterbi search over one whitespace-free stretch of text. */ + /** + * Runs the Viterbi search over one whitespace-free stretch of text. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param morphemes Receives the cheapest path's morphemes, in text order. + * @throws IllegalStateException Thrown if no path reaches the end of the stretch. + */ private void decode(String text, int from, int to, List morphemes) { final int length = to - from; // Each element heads the chain of nodes ending at that position. @@ -179,12 +193,13 @@ private void decode(String text, int from, int to, List morphemes) { } Node best = null; + long bestTotal = Long.MAX_VALUE; for (Node node = endingAt[length]; node != null; node = node.nextEndingHere) { final long total = node.pathCost + dictionary.connectionCost(node.entry.rightId(), BOUNDARY_CONTEXT); - if (best == null || total < best.pathCost - + dictionary.connectionCost(best.entry.rightId(), BOUNDARY_CONTEXT)) { + if (best == null || total < bestTotal) { best = node; + bestTotal = total; } } if (best == null) { @@ -202,7 +217,13 @@ private void decode(String text, int from, int to, List morphemes) { } } - /** Connects a candidate to the cheapest predecessor ending where it starts. */ + /** + * Connects a candidate to the cheapest predecessor ending where it starts. + * + * @param candidate The node to give a path cost and a predecessor. + * @param predecessors The head of the chain of nodes ending where the candidate + * starts, or {@code null} when it starts at the stretch start. + */ private void relax(Node candidate, Node predecessors) { if (predecessors == null) { candidate.pathCost = candidate.entry.cost() @@ -251,18 +272,31 @@ private void computeCategoryRuns(String text, int from, int to, } } - /** Gathers lexicon matches and unknown-word candidates starting at one position. */ + /** + * Gathers lexicon matches and unknown-word candidates starting at one position. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end, which no candidate may reach past. + * @param offset The candidate start, relative to {@code from}. + * @param positionCategory The category of that position, or {@code null} for a + * position inside a surrogate pair. + * @param positionRunEnd The exclusive end of the same-category run starting there, + * meaningful only when {@code positionCategory} is not + * {@code null}. + * @param candidates Receives the candidates. Must be empty on entry. + * @throws IllegalStateException Thrown if neither the lexicon, the position's + * category, nor the {@code DEFAULT} template offers a candidate. + */ private void candidates(String text, int from, int to, int offset, Category positionCategory, int positionRunEnd, List candidates) { final int position = from + offset; - final boolean[] matched = new boolean[1]; dictionary.prefixMatches(text, position, to, (length, entries) -> { - matched[0] = true; for (final WordEntry entry : entries) { candidates.add(new Node(position, position + length, entry, false)); } }); - final boolean lexiconMatch = matched[0]; + final boolean lexiconMatch = !candidates.isEmpty(); final int codePoint = text.codePointAt(position); final Category category; @@ -286,7 +320,8 @@ private void candidates(String text, int from, int to, int offset, if (candidates.isEmpty()) { // Neither the lexicon nor the character's category produced a candidate here, so a // single-character entry from the DEFAULT template keeps the lattice connected. - final List fallback = dictionary.unknownEntries("DEFAULT"); + final List fallback = + dictionary.unknownEntries(MecabDictionary.DEFAULT_CATEGORY); if (fallback != null) { for (final WordEntry entry : fallback) { candidates.add( diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index a971c586a3..59d9ab3388 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -49,11 +49,46 @@ */ public final class MecabDictionary { - /** One lexicon or unknown-word entry. */ + /** + * The category name every {@code char.def} must define; unmapped code points and + * unknown-word handling fall back to it. + */ + static final String DEFAULT_CATEGORY = "DEFAULT"; + + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** The prefix a {@code char.def} code point field carries, in either letter case. */ + private static final String HEX_PREFIX = "0x"; + + /** The separator between the two ends of a {@code char.def} code point range. */ + private static final String RANGE_SEPARATOR = ".."; + + /** The {@code char.def} field value that turns a category flag on. */ + private static final String FLAG_ON = "1"; + + /** + * One lexicon or unknown-word entry. + * + * @param leftId The left context id, an index into the connection matrix. + * @param rightId The right context id, an index into the connection matrix. + * @param cost The entry's own cost. + * @param features The entry's feature columns, in file order. + */ record WordEntry(int leftId, int rightId, int cost, List features) { } - /** One character category's unknown-word behavior from {@code char.def}. */ + /** + * One character category's unknown-word behavior from {@code char.def}. + * + * @param name The category name. + * @param invoke Whether unknown-word candidates are generated even where the lexicon + * matched. + * @param group Whether a whole run of same-category characters is offered as one + * candidate. + * @param length How many leading characters of the run are offered as candidates. + */ record Category(String name, boolean invoke, boolean group, int length) { } @@ -236,6 +271,10 @@ private void insert(int left, int right, int depth, int state) { * Finds the lowest base at which every label lands on a free slot. Labels * arrive in surface-character order, not numeric order, so the smallest and * largest label are computed rather than assumed positional. + * + * @param labels The child labels to place. + * @param labelCount How many leading elements of {@code labels} are in use. + * @return The base offset every label fits at. */ private int findBase(int[] labels, int labelCount) { int smallest = labels[0]; @@ -261,6 +300,11 @@ private int findBase(int[] labels, int labelCount) { } } + /** + * Grows the base and check arrays until a slot is addressable. + * + * @param slot The highest slot index that has to be writable. + */ private void ensureCapacity(int slot) { if (slot >= check.length) { int capacity = check.length; @@ -358,7 +402,10 @@ private void map(int from, int to, String category) { /** * Folds the recorded mappings into their lookup table. * + * @param categories The categories the {@code char.def} category section defined, + * keyed by name. * @return The table. Never {@code null}. + * @throws IOException Thrown if a mapping names a category that was never defined. */ private CategoryTable build(Map categories) throws IOException { // Cut the supplementary ranges at every boundary they introduce, so that each @@ -412,13 +459,19 @@ private CategoryTable build(Map categories) throws IOException * Resolves a mapped category name against the defined categories, so a mapping to * a name the {@code char.def} category section never defined fails at load with * the offending code point instead of falling back silently at lookup time. + * + * @param name The category name a mapping line gave. + * @param categories The defined categories, keyed by name. + * @param codePoint A code point the mapping covers, for the error message. + * @return The resolved category. Never {@code null}. + * @throws IOException Thrown if no category of that name was defined. */ - private static Category resolve(String name, Map categories, + private Category resolve(String name, Map categories, int codePoint) throws IOException { final Category category = categories.get(name); if (category == null) { throw new IOException(String.format( - "char.def maps U+%04X to the undefined category %s", codePoint, name)); + CHAR_DEF + " maps U+%04X to the undefined category %s", codePoint, name)); } return category; } @@ -455,7 +508,6 @@ interface PrefixMatchConsumer { private final DoubleArrayLexicon lexicon; private final short[] connectionCosts; private final int rightSize; - private final Map categories; private final CategoryTable categoryTable; private final Category defaultCategory; private final Map> unknownEntries; @@ -466,9 +518,8 @@ private MecabDictionary(DoubleArrayLexicon lexicon, this.lexicon = lexicon; this.connectionCosts = connectionCosts; this.rightSize = rightSize; - this.categories = categories; this.categoryTable = categoryTable; - this.defaultCategory = categories.get("DEFAULT"); + this.defaultCategory = categories.get(DEFAULT_CATEGORY); this.unknownEntries = unknownEntries; } @@ -507,23 +558,24 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } // The connection matrix is read first because its dimensions are what every // lexicon entry's context ids have to be inside of. - final List matrixLines = readLines(directory.resolve("matrix.def"), charset); - if (matrixLines.isEmpty()) { - throw new IOException("empty matrix.def under " + directory); + final List matrixLines = readLines(directory.resolve(MATRIX_DEF), charset); + final String headerLine = StringUtil.trimUnicodeWhitespace(matrixLines.get(0)); + if (headerLine.isEmpty()) { + throw new IOException("empty " + MATRIX_DEF + " under " + directory); } - final String[] header = splitWhitespace(matrixLines.get(0)); + final String[] header = splitWhitespace(headerLine); if (header.length != 2) { - throw new IOException("malformed matrix.def header: " + matrixLines.get(0)); + throw new IOException("malformed " + MATRIX_DEF + " header: " + headerLine); } - final int leftSize = parseInt(header[0], "matrix.def", 1); - final int rightSize = parseInt(header[1], "matrix.def", 1); + final int leftSize = parseInt(header[0], MATRIX_DEF, 1); + final int rightSize = parseInt(header[1], MATRIX_DEF, 1); if (leftSize < 1 || rightSize < 1) { - throw new IOException("matrix.def dimensions must be positive, got " + throw new IOException(MATRIX_DEF + " dimensions must be positive, got " + leftSize + " " + rightSize); } final long cells = (long) leftSize * rightSize; if (cells > Integer.MAX_VALUE) { - throw new IOException("matrix.def dimensions " + leftSize + " x " + rightSize + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + " overflow the addressable connection matrix"); } final short[] costs = new short[(int) cells]; @@ -534,18 +586,18 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } final String[] fields = splitWhitespace(line); if (fields.length != 3) { - throw new IOException("malformed matrix.def line " + (i + 1)); + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1)); } - final int right = parseInt(fields[0], "matrix.def", i + 1); - final int left = parseInt(fields[1], "matrix.def", i + 1); + final int right = parseInt(fields[0], MATRIX_DEF, i + 1); + final int left = parseInt(fields[1], MATRIX_DEF, i + 1); if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { - throw new IOException("malformed matrix.def line " + (i + 1) + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) + ": context ids " + right + " " + left + " are outside the declared dimensions " + leftSize + " " + rightSize); } - final int cost = parseInt(fields[2], "matrix.def", i + 1); + final int cost = parseInt(fields[2], MATRIX_DEF, i + 1); if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { - throw new IOException("malformed matrix.def line " + (i + 1) + throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) + ": connection cost " + cost + " is outside the 16-bit range the" + " format defines"); } @@ -564,10 +616,10 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc final Map categories = new HashMap<>(); final CategoryTableBuilder categoryTable = new CategoryTableBuilder(); - readCharacterDefinition(directory.resolve("char.def"), charset, categories, + readCharacterDefinition(directory.resolve(CHAR_DEF), charset, categories, categoryTable); final Map> unknown = new HashMap<>(); - readLexicon(directory.resolve("unk.def"), charset, unknown, leftSize, rightSize); + readLexicon(directory.resolve(UNK_DEF), charset, unknown, leftSize, rightSize); return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, rightSize, categories, categoryTable.build(categories), unknown); @@ -608,12 +660,12 @@ private static void readLexicon(Path file, Charset charset, final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); if (leftId < 0 || leftId >= rightSize) { throw new IOException("malformed entry at " + file + " line " + lineNumber - + ": left context id " + leftId + " is outside the matrix.def dimensions " + + ": left context id " + leftId + " is outside the " + MATRIX_DEF + " dimensions " + leftSize + " " + rightSize); } if (rightId < 0 || rightId >= leftSize) { throw new IOException("malformed entry at " + file + " line " + lineNumber - + ": right context id " + rightId + " is outside the matrix.def dimensions " + + ": right context id " + rightId + " is outside the " + MATRIX_DEF + " dimensions " + leftSize + " " + rightSize); } final WordEntry entry = new WordEntry(leftId, rightId, @@ -623,7 +675,18 @@ private static void readLexicon(Path file, Charset charset, } } - /** Reads char.def: category behavior lines and code point mapping lines. */ + /** + * Reads {@code char.def}: the category behavior lines and the code point mapping + * lines, in file order, so that a later mapping wins over an earlier one. + * + * @param file The file to read. + * @param charset The encoding to decode with. + * @param categories Receives the defined categories, keyed by name. + * @param categoryTable Receives the code point to category name mappings. + * @throws IOException Thrown if the file is missing, a line is malformed, a code + * point is outside the Unicode range, a range descends, or the file defines + * no {@code DEFAULT} category. + */ private static void readCharacterDefinition(Path file, Charset charset, Map categories, CategoryTableBuilder categoryTable) throws IOException { @@ -635,13 +698,14 @@ private static void readCharacterDefinition(Path file, Charset charset, continue; } final String[] fields = splitWhitespace(line); - if (fields[0].startsWith("0x") || fields[0].startsWith("0X")) { - final int rangeSeparator = fields[0].indexOf(".."); + if (fields[0].regionMatches(true, 0, HEX_PREFIX, 0, HEX_PREFIX.length())) { + final int rangeSeparator = fields[0].indexOf(RANGE_SEPARATOR); final int from; final int to; if (rangeSeparator >= 0) { from = parseCodePoint(fields[0].substring(0, rangeSeparator), file, lineNumber); - to = parseCodePoint(fields[0].substring(rangeSeparator + 2), file, lineNumber); + to = parseCodePoint( + fields[0].substring(rangeSeparator + RANGE_SEPARATOR.length()), file, lineNumber); } else { from = parseCodePoint(fields[0], file, lineNumber); to = from; @@ -659,12 +723,12 @@ private static void readCharacterDefinition(Path file, Charset charset, throw new IOException("malformed category at " + file + " line " + lineNumber); } categories.put(fields[0], new Category(fields[0], - "1".equals(fields[1]), "1".equals(fields[2]), + FLAG_ON.equals(fields[1]), FLAG_ON.equals(fields[2]), parseInt(fields[3], file.toString(), lineNumber))); } } - if (!categories.containsKey("DEFAULT")) { - throw new IOException("char.def defines no DEFAULT category: " + file); + if (!categories.containsKey(DEFAULT_CATEGORY)) { + throw new IOException(CHAR_DEF + " defines no " + DEFAULT_CATEGORY + " category: " + file); } } @@ -721,7 +785,8 @@ List unknownEntries(String category) { * * @param file The file to read. * @param charset The encoding to decode with. - * @return The lines without their line terminators. Never {@code null}. + * @return The lines without their line terminators. Never {@code null} and never + * empty: an empty file reads as one empty line. * @throws IOException Thrown if the file is missing or reading fails. */ private static List readLines(Path file, Charset charset) throws IOException { @@ -822,14 +887,15 @@ private static int parseInt(String text, String file, int lineNumber) * @param file The file being read, for the error message. * @param lineNumber The line being read, for the error message. * @return The parsed code point, which may be in a supplementary plane. - * @throws IOException Thrown if the field is not a valid hexadecimal code point or - * names a value no Unicode code point has. + * @throws IOException Thrown if the field is shorter than the prefix, is not a valid + * hexadecimal number, or names a value no Unicode code point has. */ private static int parseCodePoint(String text, Path file, int lineNumber) throws IOException { final int codePoint; try { - codePoint = Integer.parseInt(StringUtil.trimUnicodeWhitespace(text).substring(2), 16); + codePoint = Integer.parseInt( + StringUtil.trimUnicodeWhitespace(text).substring(HEX_PREFIX.length()), 16); } catch (RuntimeException e) { throw new IOException("malformed code point in " + file + " line " + lineNumber, e); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 3c2589bc02..76212b1b5d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -246,7 +246,15 @@ private static void skip(InputStream in, long bytes) throws IOException { } } - /** Wraps the tar stream so exactly one entry's bytes are readable. */ + /** + * Wraps the tar stream so exactly one entry's bytes are readable. + * + * @param in The tar stream, positioned at the entry's first byte. + * @param size The entry size in bytes. + * @return A stream reporting end of stream after that many bytes, and failing if the + * tar stream ends first. Never {@code null}; closing it leaves {@code in} + * open and positioned after the entry content. + */ private static InputStream boundedStream(InputStream in, long size) { return new InputStream() { private long remaining = size; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index 00469ac4f7..060a0a53d2 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -183,14 +183,14 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) * *

Reports the segmented surfaces, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. */ @Override - public String[] tokenize(String s) { - final Span[] spans = tokenizePos(s); + public String[] tokenize(String text) { + final Span[] spans = tokenizePos(text); final String[] tokens = new String[spans.length]; for (int i = 0; i < tokens.length; i++) { - tokens[i] = s.substring(spans[i].getStart(), spans[i].getEnd()); + tokens[i] = text.substring(spans[i].getStart(), spans[i].getEnd()); } return tokens; } @@ -200,31 +200,39 @@ public String[] tokenize(String s) { * *

Reports the segmented spans in original text coordinates, whitespace omitted.

* - * @throws IllegalArgumentException Thrown if {@code s} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. */ @Override - public Span[] tokenizePos(String s) { - if (s == null) { + public Span[] tokenizePos(String text) { + if (text == null) { throw new IllegalArgumentException("text must not be null"); } final List spans = new ArrayList<>(); int start = 0; - while (start < s.length()) { - if (StringUtil.isWhitespace(s.charAt(start))) { + while (start < text.length()) { + if (StringUtil.isWhitespace(text.charAt(start))) { start++; continue; } int end = start; - while (end < s.length() && !StringUtil.isWhitespace(s.charAt(end))) { + while (end < text.length() && !StringUtil.isWhitespace(text.charAt(end))) { end++; } - decode(s, start, end, spans); + decode(text, start, end, spans); start = end; } return spans.toArray(new Span[0]); } - /** Viterbi over word log-probabilities within one whitespace-free stretch. */ + /** + * Viterbi over word log-probabilities within one whitespace-free stretch. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param spans Receives the best path's spans, in text order and in original text + * coordinates. + */ private void decode(String text, int from, int to, List spans) { final int length = to - from; final double[] best = new double[length + 1]; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 034cd6d8c3..56a3ea8710 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -21,12 +21,15 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.Span; @@ -40,6 +43,24 @@ */ public class LatticeTokenizerTest { + private static final String LEXICON_CSV = "lexicon.csv"; + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** A one by one connection matrix charging cost zero, for single-context fixtures. */ + private static final String UNIT_MATRIX = "1 1\n0 0 0\n"; + + /** + * The {@code char.def} line defining the DEFAULT category: it does not invoke + * unknown-word handling beside a lexicon match, it groups a whole run into one + * candidate, and it offers no fixed-length candidates. + */ + private static final String DEFAULT_CATEGORY_LINE = "DEFAULT 0 1 0"; + + /** The {@code unk.def} template line for the DEFAULT category. */ + private static final String DEFAULT_UNKNOWN_TEMPLATE = "DEFAULT,0,0,10000,symbol,unknown"; + @TempDir static Path directory; @@ -47,7 +68,7 @@ public class LatticeTokenizerTest { @BeforeAll static void loadDictionary() throws IOException { - write("lexicon.csv", String.join("\n", + write(LEXICON_CSV, String.join("\n", "\u6771\u4EAC,0,0,3000,noun,proper", "\u4EAC\u90FD,0,0,3000,noun,proper", "\u6771,0,0,6000,noun,common", @@ -55,9 +76,9 @@ static void loadDictionary() throws IOException { "\u306B,0,0,1000,particle,case", "\u884C\u304F,0,0,3000,verb,base", "")); - write("matrix.def", "1 1\n0 0 0\n"); - write("char.def", String.join("\n", - "DEFAULT 0 1 0", + write(MATRIX_DEF, UNIT_MATRIX); + write(CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "KANJI 0 0 2", "HIRAGANA 0 1 0", "LATIN 1 1 0", @@ -67,8 +88,8 @@ static void loadDictionary() throws IOException { "0x0041..0x005A LATIN", "0x0061..0x007A LATIN", "")); - write("unk.def", String.join("\n", - "DEFAULT,0,0,10000,symbol,unknown", + write(UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, "LATIN,0,0,4000,noun,foreign", "KANJI,0,0,8000,noun,unknown", "HIRAGANA,0,0,9000,particle,unknown", @@ -76,13 +97,19 @@ static void loadDictionary() throws IOException { tokenizer = new LatticeTokenizer(MecabDictionary.load(directory)); } + /** Writes one UTF-8 dictionary file into the shared dictionary directory. */ private static void write(String name, String content) throws IOException { - Files.write(directory.resolve(name), content.getBytes(StandardCharsets.UTF_8)); + write(directory, name, content); + } + + /** Writes one UTF-8 dictionary file into a test-supplied directory. */ + private static void write(Path target, String name, String content) throws IOException { + Files.write(target.resolve(name), content.getBytes(StandardCharsets.UTF_8)); } @Test void testLatticePrefersTheCheaperSegmentation() { - // the famous case: Tokyo+Metro must win over East+Kyoto + // Tokyo plus the metropolis suffix must beat the competing reading east plus Kyoto. final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; Assertions.assertArrayEquals( new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, @@ -249,8 +276,7 @@ void testSpansStayOriginalAfterLeadingWhitespace() { void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { // The rest of the dictionary is well formed, so the short row is what load rejects. writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -262,8 +288,7 @@ void testShortLexiconRowFailsLoud(@TempDir Path broken) throws IOException { void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException { // The rest of the dictionary is well formed, so the cost column is what load rejects. writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,abc,noun\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0,abc,noun\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -273,9 +298,8 @@ void testNonNumericLexiconCostFailsLoud(@TempDir Path broken) throws IOException */ @Test void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("matrix.def"), "1 1\n0 0\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, "1 1\n0 0\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -286,11 +310,9 @@ void testMalformedMatrixLineFailsLoud(@TempDir Path broken) throws IOException { @Test void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) throws IOException { - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("char.def"), - "DEFAULT 0 1 0\n0x4E00..0x9FFF\n".getBytes(StandardCharsets.UTF_8)); + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n0x4E00..0x9FFF\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -303,30 +325,90 @@ void testCharDefMappingWithoutCategoryFailsLoud(@TempDir Path broken) @Test void testMissingDefaultTemplateFailsLoudAtTokenizeTime(@TempDir Path partial) throws IOException { - Files.write(partial.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(partial.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(partial.resolve("char.def"), - "DEFAULT 0 1 0\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n" - .getBytes(StandardCharsets.UTF_8)); - Files.write(partial.resolve("unk.def"), - "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + write(partial, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(partial, MATRIX_DEF, UNIT_MATRIX); + write(partial, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\nKANJI 0 0 2\n0x4E00..0x9FFF KANJI\n"); + write(partial, UNK_DEF, "KANJI,0,0,8000,noun\n"); final LatticeTokenizer limited = new LatticeTokenizer(MecabDictionary.load(partial)); Assertions.assertThrows(IllegalStateException.class, () -> limited.analyze("\u2460")); } + /** + * Verifies that a directory holding a lexicon but none of the definition files is + * rejected at load time, naming the first file that is missing. + */ @Test - void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException { - Files.write(broken.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); - Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); + void testMissingDefinitionFileFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("required dictionary file is missing: " + + broken.resolve(MATRIX_DEF), e.getMessage()); + } + + /** + * Verifies that a {@code char.def} without the mandatory DEFAULT category is rejected + * at load time rather than leaving unmapped code points without a fallback. + */ + @Test + void testCharDefWithoutDefaultCategoryFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n"); + write(broken, UNK_DEF, "KANJI,0,0,8000,noun\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("char.def defines no DEFAULT category: " + + broken.resolve(CHAR_DEF), e.getMessage()); + } + + /** + * Verifies that a directory with the definition files but no lexicon entry at all is + * rejected at load time, since no text could be segmented against it. + */ + @Test + void testDictionaryWithoutLexiconEntriesFailsLoud(@TempDir Path empty) throws IOException { + writeUnitMatrixDictionary(empty); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(empty)); + Assertions.assertEquals("no lexicon entries found under " + empty, e.getMessage()); + } + + /** + * Verifies that an empty {@code matrix.def} is reported as such instead of as a + * malformed header with nothing to show. + */ + @Test + void testEmptyMatrixDefFailsLoud(@TempDir Path broken) throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, ""); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("empty matrix.def under " + broken, e.getMessage()); + } - Files.write(broken.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("char.def"), - "KANJI 0 0 2\n0x4E00..0x9FFF KANJI\n".getBytes(StandardCharsets.UTF_8)); - Files.write(broken.resolve("unk.def"), - "KANJI,0,0,8000,noun\n".getBytes(StandardCharsets.UTF_8)); + /** + * Verifies the {@code char.def} fail-loud paths that a malformed line can take: a + * descending code point range, a code point outside the Unicode range, a code point + * field that is not hexadecimal, and a category line missing its length column. + * + * @param charDef The {@code char.def} content under test. + * @param broken The directory the fixture dictionary is written into. + * @throws IOException Thrown if writing the fixture fails. + */ + @ParameterizedTest(name = "[{index}] char.def {0}") + @ValueSource(strings = { + DEFAULT_CATEGORY_LINE + "\n0x0110..0x0100 LATIN\n", + DEFAULT_CATEGORY_LINE + "\n0x110000 LATIN\n", + DEFAULT_CATEGORY_LINE + "\n0xZZ LATIN\n", + "DEFAULT 0 1\n"}) + void testMalformedCharDefFailsLoud(String charDef, @TempDir Path broken) + throws IOException { + write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(broken, MATRIX_DEF, UNIT_MATRIX); + write(broken, CHAR_DEF, charDef); + write(broken, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } @@ -339,23 +421,22 @@ void testMalformedDictionariesFailLoud(@TempDir Path broken) throws IOException * @throws IOException Thrown if writing any of the files fails. */ private static void writeSupplementaryDictionary(Path target) throws IOException { - Files.write(target.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun,common\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(target, LEXICON_CSV, "\u6771,0,0,6000,noun,common\n"); + write(target, MATRIX_DEF, UNIT_MATRIX); + write(target, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "KANJI 0 0 2", "LATIN 1 1 0", "", "0x4E00..0x9FFF KANJI", "0x20000..0x2A6DF KANJI", "0x0061..0x007A LATIN", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("unk.def"), String.join("\n", - "DEFAULT,0,0,10000,symbol,unknown", + "")); + write(target, UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, "KANJI,0,0,8000,noun,unknown", "LATIN,0,0,4000,noun,foreign", - "").getBytes(StandardCharsets.UTF_8)); + "")); } /** @@ -432,11 +513,9 @@ void testSupplementaryIdeographDoesNotAbsorbItsNeighbour(@TempDir Path supplemen * @throws IOException Thrown if writing any of the files fails. */ private static void writeUnitMatrixDictionary(Path target) throws IOException { - Files.write(target.resolve("matrix.def"), "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("char.def"), - "DEFAULT 0 1 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(target.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + write(target, MATRIX_DEF, UNIT_MATRIX); + write(target, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n"); + write(target, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); } /** @@ -449,11 +528,10 @@ private static void writeUnitMatrixDictionary(Path target) throws IOException { void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) throws IOException { writeUnitMatrixDictionary(mismatched); - Files.write(mismatched.resolve("lexicon.csv"), - "\u6771,0,5,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + write(mismatched, LEXICON_CSV, "\u6771,0,5,3000,noun\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(mismatched)); - Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + Assertions.assertEquals("malformed entry at " + mismatched.resolve(LEXICON_CSV) + " line 1: right context id 5 is outside the matrix.def dimensions 1 1", e.getMessage()); } @@ -467,11 +545,10 @@ void testRightContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) void testLeftContextIdBeyondMatrixFailsLoudAtLoad(@TempDir Path mismatched) throws IOException { writeUnitMatrixDictionary(mismatched); - Files.write(mismatched.resolve("lexicon.csv"), - "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n".getBytes(StandardCharsets.UTF_8)); + write(mismatched, LEXICON_CSV, "\u6771,0,0,3000,noun\n\u90FD,7,0,3000,noun\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(mismatched)); - Assertions.assertEquals("malformed entry at " + mismatched.resolve("lexicon.csv") + Assertions.assertEquals("malformed entry at " + mismatched.resolve(LEXICON_CSV) + " line 2: left context id 7 is outside the matrix.def dimensions 1 1", e.getMessage()); } @@ -482,7 +559,38 @@ void testInvalidArguments() { () -> new LatticeTokenizer(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> MecabDictionary.load(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionary.load(directory, null)); Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.analyze(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> tokenizer.tokenize(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> tokenizer.tokenizePos(null)); + } + + /** + * Verifies the {@link Morpheme} contract every segmentation result is built from: a + * {@code null} span, a {@code null} or empty surface, and {@code null} features are + * all rejected, and the feature list is copied so a later change to the caller's list + * cannot be seen through the morpheme. + */ + @Test + void testMorphemeRejectsInvalidArguments() { + final Span span = new Span(0, 1); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(null, "東", List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, null, List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, "", List.of("noun"), false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new Morpheme(span, "東", null, false)); + + final List features = new ArrayList<>(List.of("noun")); + final Morpheme morpheme = new Morpheme(span, "東", features, false); + features.add("proper"); + Assertions.assertEquals(List.of("noun"), morpheme.features()); } /** @@ -494,20 +602,17 @@ void testInvalidArguments() { @Test void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlapped) throws IOException { - Files.write(overlapped.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(overlapped.resolve("matrix.def"), - "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(overlapped.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(overlapped, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(overlapped, MATRIX_DEF, UNIT_MATRIX); + write(overlapped, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "KANJI 0 0 2", "LATIN 1 1 0", "", "0x20000..0x2FFFF KANJI", "0x24000..0x25000 LATIN", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(overlapped.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + "")); + write(overlapped, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); final MecabDictionary dictionary = MecabDictionary.load(overlapped); Assertions.assertEquals("KANJI", dictionary.categoryOf(0x20000).name()); @@ -527,18 +632,15 @@ void testLaterSupplementaryMappingWinsInsideAnEarlierRange(@TempDir Path overlap @Test void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) throws IOException { - Files.write(straddling.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(straddling.resolve("matrix.def"), - "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(straddling.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(straddling, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(straddling, MATRIX_DEF, UNIT_MATRIX); + write(straddling, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "LATIN 1 1 0", "", "0xFF00..0x10040 LATIN", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(straddling.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + "")); + write(straddling, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); final MecabDictionary dictionary = MecabDictionary.load(straddling); Assertions.assertEquals("LATIN", dictionary.categoryOf(0xFF00).name()); @@ -556,17 +658,14 @@ void testCharDefRangeStraddlingTheBmpBoundary(@TempDir Path straddling) */ @Test void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOException { - Files.write(ghost.resolve("lexicon.csv"), - "\u6771,0,0,6000,noun\n".getBytes(StandardCharsets.UTF_8)); - Files.write(ghost.resolve("matrix.def"), - "1 1\n0 0 0\n".getBytes(StandardCharsets.UTF_8)); - Files.write(ghost.resolve("char.def"), String.join("\n", - "DEFAULT 0 1 0", + write(ghost, LEXICON_CSV, "\u6771,0,0,6000,noun\n"); + write(ghost, MATRIX_DEF, UNIT_MATRIX); + write(ghost, CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, "", "0x0100..0x0110 GHOST", - "").getBytes(StandardCharsets.UTF_8)); - Files.write(ghost.resolve("unk.def"), - "DEFAULT,0,0,10000,symbol,unknown\n".getBytes(StandardCharsets.UTF_8)); + "")); + write(ghost, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(ghost)); @@ -582,8 +681,7 @@ void testMappingToUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOExcep @Test void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOException { writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("matrix.def"), - "1 1\n0 0 40000\n".getBytes(StandardCharsets.UTF_8)); + write(broken, MATRIX_DEF, "1 1\n0 0 40000\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Assertions.assertEquals("malformed matrix.def line 2: connection cost 40000 is" @@ -599,8 +697,7 @@ void testMatrixCostOutsideShortRangeFailsLoud(@TempDir Path broken) throws IOExc void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) throws IOException { writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("matrix.def"), - "70000 70000\n".getBytes(StandardCharsets.UTF_8)); + write(broken, MATRIX_DEF, "70000 70000\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Assertions.assertEquals("matrix.def dimensions 70000 x 70000 overflow the" @@ -615,8 +712,7 @@ void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) void testMatrixRowContextIdsOutsideDimensionsFailLoud(@TempDir Path broken) throws IOException { writeUnitMatrixDictionary(broken); - Files.write(broken.resolve("matrix.def"), - "1 1\n2 0 5\n".getBytes(StandardCharsets.UTF_8)); + write(broken, MATRIX_DEF, "1 1\n2 0 5\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); Assertions.assertEquals("malformed matrix.def line 2: context ids 2 0 are outside" diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index da95c97366..fb5c6a701f 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -85,6 +85,8 @@ void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) void testInvalidArguments(@TempDir Path target) { Assertions.assertThrows(IllegalArgumentException.class, () -> MecabDictionaryInstaller.install(null, target)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.install(target.toUri(), null)); Assertions.assertThrows(IllegalArgumentException.class, () -> MecabDictionaryInstaller.extract(null, target)); Assertions.assertThrows(IllegalArgumentException.class, diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java index 4fdfb29d01..896115aba2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java @@ -28,6 +28,21 @@ */ final class TarGzArchives { + /** The tar block size; headers, content, and padding are all whole blocks. */ + private static final int BLOCK = 512; + + /** The header field offsets and lengths this builder writes, in bytes. */ + private static final int NAME_LENGTH = 100; + private static final int MODE_OFFSET = 100; + private static final int SIZE_OFFSET = 124; + private static final int SIZE_LENGTH = 12; + private static final int CHECKSUM_OFFSET = 148; + private static final int CHECKSUM_LENGTH = 8; + private static final int TYPE_OFFSET = 156; + + /** The type flag of a regular file entry. */ + private static final char REGULAR_FILE = '0'; + private TarGzArchives() { } @@ -45,7 +60,8 @@ static byte[] gzippedTar(String[][] entries) throws IOException { for (final String[] entry : entries) { tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); } - tar.write(new byte[1024]); + // Two zero blocks end a tar archive. + tar.write(new byte[2 * BLOCK]); final ByteArrayOutputStream compressed = new ByteArrayOutputStream(); try (GZIPOutputStream gzip = new GZIPOutputStream(compressed)) { gzip.write(tar.toByteArray()); @@ -67,31 +83,35 @@ static byte[] gzippedTar(String[][] entries) throws IOException { private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) throws IOException { final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); - if (nameBytes.length == 0 || nameBytes.length > 100) { + if (nameBytes.length == 0 || nameBytes.length > NAME_LENGTH) { throw new IllegalArgumentException( - "entry name must be 1..100 bytes, got " + nameBytes.length); + "entry name must be 1.." + NAME_LENGTH + " bytes, got " + nameBytes.length); } - final byte[] header = new byte[512]; + final byte[] header = new byte[BLOCK]; System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); - System.arraycopy(mode, 0, header, 100, mode.length); - final String size = String.format("%011o", content.length); - System.arraycopy(size.getBytes(StandardCharsets.US_ASCII), 0, header, 124, 11); - header[156] = '0'; - for (int i = 148; i < 156; i++) { + System.arraycopy(mode, 0, header, MODE_OFFSET, mode.length); + // Both numeric fields hold octal digits followed by one terminator byte. + final byte[] size = String.format("%0" + (SIZE_LENGTH - 1) + "o", content.length) + .getBytes(StandardCharsets.US_ASCII); + System.arraycopy(size, 0, header, SIZE_OFFSET, size.length); + header[TYPE_OFFSET] = REGULAR_FILE; + // The checksum is computed with its own field read as spaces. + for (int i = CHECKSUM_OFFSET; i < CHECKSUM_OFFSET + CHECKSUM_LENGTH; i++) { header[i] = ' '; } int checksum = 0; for (final byte b : header) { checksum += b & 0xFF; } - final String checksumText = String.format("%06o", checksum); - System.arraycopy(checksumText.getBytes(StandardCharsets.US_ASCII), 0, header, 148, 6); - header[154] = 0; - header[155] = ' '; + final byte[] checksumText = String.format("%0" + (CHECKSUM_LENGTH - 2) + "o", checksum) + .getBytes(StandardCharsets.US_ASCII); + System.arraycopy(checksumText, 0, header, CHECKSUM_OFFSET, checksumText.length); + header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 2] = 0; + header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 1] = ' '; tar.write(header); tar.write(content); - final int padding = (512 - content.length % 512) % 512; + final int padding = (BLOCK - content.length % BLOCK) % BLOCK; tar.write(new byte[padding]); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java index 977d42fcc4..487c1c6ad3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/UnigramSegmenterTest.java @@ -19,11 +19,15 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.util.Span; @@ -153,26 +157,28 @@ void testSpansStayOriginalAfterLeadingWhitespace() { segmenter.tokenizePos(text)); } - @Test - void testMalformedLexiconsFailLoud() { - Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("word\n".getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)); - Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("word abc\n".getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)); - Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("word 0\n".getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8)); + @ParameterizedTest(name = "lexicon content \"{0}\"") + @ValueSource(strings = {"word\n", "word abc\n", "word 0\n", "\n\n"}) + void testMalformedLexiconsFailLoud(String lexicon) { Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( - new ByteArrayInputStream("\n\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); } @Test void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, - () -> UnigramSegmenter.load((java.nio.file.Path) null)); + () -> UnigramSegmenter.load((Path) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((Path) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(Path.of("words.txt"), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((InputStream) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(new ByteArrayInputStream(new byte[0]), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenize(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> segmenter.tokenizePos(null)); } From 263035581995431c81bbe9283ffe9f160517a3e6 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 07:00:37 -0400 Subject: [PATCH 15/24] OPENNLP-1894: Move lattice tokenizer types into opennlp-api Place LatticeTokenizer, UnigramSegmenter, MecabDictionary, and Morpheme with the other resource-driven tokenizers. Keep MecabDictionaryInstaller in runtime. Lift MAX_ENTRIES into ResourceLimits so api loaders can share the bound, and reject incomplete or oversized matrix.def payloads. --- .../tokenize/lattice/LatticeTokenizer.java | 0 .../tokenize/lattice/MecabDictionary.java | 55 +++++++++++++++--- .../tools/tokenize/lattice/Morpheme.java | 0 .../tokenize/lattice/UnigramSegmenter.java | 0 .../opennlp/tools/util/ResourceLimits.java | 57 +++++++++++++++++++ .../tools/ml/model/AbstractModelReader.java | 24 ++------ .../lattice/LatticeTokenizerTest.java | 46 +++++++++++++++ opennlp-docs/src/docbkx/tokenizer.xml | 3 + 8 files changed, 160 insertions(+), 25 deletions(-) rename {opennlp-core/opennlp-runtime => opennlp-api}/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java (100%) rename {opennlp-core/opennlp-runtime => opennlp-api}/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java (93%) rename {opennlp-core/opennlp-runtime => opennlp-api}/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java (100%) rename {opennlp-core/opennlp-runtime => opennlp-api}/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java (100%) create mode 100644 opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java similarity index 100% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java rename to opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/LatticeTokenizer.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java similarity index 93% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java rename to opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 59d9ab3388..d422685710 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -25,10 +25,13 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import opennlp.tools.util.ResourceLimits; import opennlp.tools.util.StringUtil; /** @@ -42,6 +45,13 @@ * distributions and the Korean mecab-ko-dic all load through this one reader, with the * feature columns passed through untouched because their schemas differ.

* + *

Each instance keeps about 0.75 MB of category tables keyed by the 16-bit code-unit + * space, so load once and share. Lexicon CSV files under the dictionary directory are + * read in sorted path order so tie-breaking is stable across file systems. Connection + * costs must cover every declared matrix cell; missing pairs are rejected rather than + * treated as cost zero. Matrix dimensions, the cell count, and the lexicon entry count + * are each bounded by {@link ResourceLimits#MAX_ENTRIES}.

+ * *

Instances are immutable and safe to share between threads.

* * @see LatticeTokenizer @@ -573,12 +583,25 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc throw new IOException(MATRIX_DEF + " dimensions must be positive, got " + leftSize + " " + rightSize); } + if (leftSize > ResourceLimits.MAX_ENTRIES + || rightSize > ResourceLimits.MAX_ENTRIES) { + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + + " exceed safe limit of " + ResourceLimits.MAX_ENTRIES); + } final long cells = (long) leftSize * rightSize; if (cells > Integer.MAX_VALUE) { throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + " overflow the addressable connection matrix"); } - final short[] costs = new short[(int) cells]; + if (cells > ResourceLimits.MAX_ENTRIES) { + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + + " exceed safe limit of " + ResourceLimits.MAX_ENTRIES); + } + final int cellCount = (int) cells; + final short[] costs = new short[cellCount]; + // leftSize bounds right-context ids and rightSize bounds left-context ids, matching + // MeCab's connector.h layout (the names read transposed against the id names). + final BitSet filled = new BitSet(cellCount); for (int i = 1; i < matrixLines.size(); i++) { final String line = StringUtil.trimUnicodeWhitespace(matrixLines.get(i)); if (line.isEmpty()) { @@ -601,15 +624,27 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc + ": connection cost " + cost + " is outside the 16-bit range the" + " format defines"); } - costs[right * rightSize + left] = (short) cost; + final int index = right * rightSize + left; + costs[index] = (short) cost; + filled.set(index); + } + if (filled.cardinality() != cellCount) { + throw new IOException(MATRIX_DEF + " declares " + leftSize + " x " + rightSize + + " connection costs but only " + filled.cardinality() + " pairs are listed"); } final Map> lexicon = new HashMap<>(); - try (DirectoryStream csvFiles = Files.newDirectoryStream(directory, "*.csv")) { - for (final Path csv : csvFiles) { - readLexicon(csv, charset, lexicon, leftSize, rightSize); + final List csvFiles = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(directory, "*.csv")) { + for (final Path csv : stream) { + csvFiles.add(csv); } } + Collections.sort(csvFiles); + final int[] entryCount = {0}; + for (final Path csv : csvFiles) { + readLexicon(csv, charset, lexicon, leftSize, rightSize, entryCount); + } if (lexicon.isEmpty()) { throw new IOException("no lexicon entries found under " + directory); } @@ -619,7 +654,8 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc readCharacterDefinition(directory.resolve(CHAR_DEF), charset, categories, categoryTable); final Map> unknown = new HashMap<>(); - readLexicon(directory.resolve(UNK_DEF), charset, unknown, leftSize, rightSize); + readLexicon(directory.resolve(UNK_DEF), charset, unknown, leftSize, rightSize, + new int[] {0}); return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, rightSize, categories, categoryTable.build(categories), unknown); @@ -640,7 +676,7 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc * entry's context id is outside the matrix dimensions. */ private static void readLexicon(Path file, Charset charset, - Map> target, int leftSize, int rightSize) + Map> target, int leftSize, int rightSize, int[] entryCount) throws IOException { int lineNumber = 0; for (final String line : readLines(file, charset)) { @@ -668,6 +704,11 @@ private static void readLexicon(Path file, Charset charset, + ": right context id " + rightId + " is outside the " + MATRIX_DEF + " dimensions " + leftSize + " " + rightSize); } + if (entryCount[0] >= ResourceLimits.MAX_ENTRIES) { + throw new IOException("lexicon entry count exceeds safe limit of " + + ResourceLimits.MAX_ENTRIES); + } + entryCount[0]++; final WordEntry entry = new WordEntry(leftId, rightId, parseInt(fields.get(3), file.toString(), lineNumber), List.copyOf(fields.subList(4, fields.size()))); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java similarity index 100% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java rename to opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/Morpheme.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java similarity index 100% rename from opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java rename to opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java diff --git a/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java b/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java new file mode 100644 index 0000000000..5b79fbe955 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +/** + * Shared upper bounds for counts read from user-supplied resources, so a crafted + * file cannot force an outsized allocation before validation completes. + */ +public final class ResourceLimits { + + /** + * System property for overriding {@link #MAX_ENTRIES}. + * Set at JVM startup, e.g. {@code -DOPENNLP_MAX_ENTRIES=5000000}. + * Falls back to {@code 10_000_000} if absent or invalid. + */ + public static final String MAX_ENTRIES_PROPERTY = "OPENNLP_MAX_ENTRIES"; + + /** + * Upper bound on count fields and resource sizes that drive allocations + * (matrix dimensions, lexicon entries, model outcome counts, and similar). + * Configurable via {@link #MAX_ENTRIES_PROPERTY}. + */ + public static final int MAX_ENTRIES = initMaxEntries(); + + private ResourceLimits() { + } + + private static int initMaxEntries() { + final String prop = System.getProperty(MAX_ENTRIES_PROPERTY, "").trim(); + if (!prop.isEmpty()) { + try { + final int val = Integer.parseInt(prop); + if (val > 0) { + return val; + } + } catch (NumberFormatException ignore) { + // Fall through to the default. + } + } + return 10_000_000; + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java index 8325b04f21..c24349401f 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java @@ -24,6 +24,8 @@ import java.util.StringTokenizer; import java.util.zip.GZIPInputStream; +import opennlp.tools.util.ResourceLimits; + /** * An abstract, basic implementation of a model reader. */ @@ -32,31 +34,17 @@ public abstract class AbstractModelReader { /** * System property for overriding the maximum number of entries (outcomes, predicates, * outcome patterns, chunk counts) that may be read from a model file or training data. - * Set at JVM startup, e.g. {@code -DOPENNLP_MAX_ENTRIES=5000000}. - * Falls back to {@code 10_000_000} if absent or invalid. + * Alias of {@link ResourceLimits#MAX_ENTRIES_PROPERTY}. */ - public static final String MAX_ENTRIES_PROPERTY = "OPENNLP_MAX_ENTRIES"; + public static final String MAX_ENTRIES_PROPERTY = ResourceLimits.MAX_ENTRIES_PROPERTY; /** * Upper bound on count fields read from a model file. - * Prevents OOM on crafted inputs with oversized array size declarations. - * Configurable via the {@link #MAX_ENTRIES_PROPERTY} system property. - *

+ * Alias of {@link ResourceLimits#MAX_ENTRIES}. * Public so that deserializers outside this package which implement their own binary * format can apply the same bound to their count fields. */ - public static final int MAX_ENTRIES = initMaxEntries(); - - private static int initMaxEntries() { - String prop = System.getProperty(MAX_ENTRIES_PROPERTY, "").trim(); - if (!prop.isEmpty()) { - try { - int val = Integer.parseInt(prop); - if (val > 0) return val; - } catch (NumberFormatException ignore) { } - } - return 10_000_000; - } + public static final int MAX_ENTRIES = ResourceLimits.MAX_ENTRIES; /** * The number of predicates contained in a model. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 56a3ea8710..b5e6c0722a 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.util.ResourceLimits; import opennlp.tools.util.Span; /** @@ -704,6 +705,51 @@ void testMatrixDimensionProductBeyondIntRangeFailsLoud(@TempDir Path broken) + " addressable connection matrix", e.getMessage()); } + /** + * Verifies that a single matrix dimension above {@link ResourceLimits#MAX_ENTRIES} + * is rejected before the connection-cost array is allocated. + */ + @Test + void testMatrixDimensionAboveMaxEntriesFailsLoud(@TempDir Path broken) throws IOException { + writeUnitMatrixDictionary(broken); + final int over = ResourceLimits.MAX_ENTRIES + 1; + write(broken, MATRIX_DEF, over + " 1\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertTrue(e.getMessage().contains("exceed safe limit of " + + ResourceLimits.MAX_ENTRIES), e.getMessage()); + } + + /** + * Verifies that a matrix whose cell count is above {@link ResourceLimits#MAX_ENTRIES} + * but still below {@link Integer#MAX_VALUE} is rejected. Without that bound, a header + * such as {@code 46340 46340} would allocate about 4 GiB of shorts. + */ + @Test + void testMatrixCellCountAboveMaxEntriesFailsLoud(@TempDir Path broken) throws IOException { + writeUnitMatrixDictionary(broken); + // 5000 x 5000 = 25_000_000 cells, above the default MAX_ENTRIES of 10_000_000. + write(broken, MATRIX_DEF, "5000 5000\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("matrix.def dimensions 5000 x 5000 exceed safe limit of " + + ResourceLimits.MAX_ENTRIES, e.getMessage()); + } + + /** + * Verifies that a truncated {@code matrix.def} fails loud. Unlisted pairs must not + * keep the short-array default of cost zero, the cheapest connection. + */ + @Test + void testIncompleteMatrixFailsLoud(@TempDir Path broken) throws IOException { + writeUnitMatrixDictionary(broken); + write(broken, MATRIX_DEF, "2 2\n0 0 1\n0 1 2\n1 0 3\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(broken)); + Assertions.assertEquals("matrix.def declares 2 x 2 connection costs but only 3" + + " pairs are listed", e.getMessage()); + } + /** * Verifies that a {@code matrix.def} data row naming context ids outside the * declared dimensions is rejected at load with the offending line and ids. diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index 6f74b6a601..f9f5466a46 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -547,6 +547,9 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { UnigramSegmenter does the same from a plain frequency lexicon. Install a dictionary archive with MecabDictionaryInstaller, load it as a MecabDictionary, and tokenize. + matrix.def must list a cost for every declared cell, and matrix + dimensions plus lexicon size are bounded by the shared + ResourceLimits.MAX_ENTRIES limit. LatticeUsageExampleTest asserts the install-load-tokenize and lexicon flows shown here. From b79cc31043d7ca8e7e77b98f2e0ebf08c942746d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 08:30:49 -0400 Subject: [PATCH 16/24] OPENNLP-1894: Cap archive extract budgets and harden MeCab load edges Bound per-entry size, total bytes, entry count, and gzip expansion during install. Fail loud on undefined unk.def categories; accept quoted CSV fields. Pin with installer and load tests; note the limits in the manual. --- .../tokenize/lattice/MecabDictionary.java | 403 ++++++++++-------- .../tokenize/lattice/UnigramSegmenter.java | 81 +++- .../lattice/MecabDictionaryInstaller.java | 225 ++++++++-- .../lattice/LatticeTokenizerTest.java | 41 +- .../lattice/MecabDictionaryInstallerTest.java | 71 +++ .../tools/tokenize/lattice/TarGzArchives.java | 122 +++++- opennlp-docs/src/docbkx/tokenizer.xml | 3 + 7 files changed, 698 insertions(+), 248 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index d422685710..dddbfc1296 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -17,6 +17,7 @@ package opennlp.tools.tokenize.lattice; +import java.io.BufferedReader; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -50,7 +51,9 @@ * read in sorted path order so tie-breaking is stable across file systems. Connection * costs must cover every declared matrix cell; missing pairs are rejected rather than * treated as cost zero. Matrix dimensions, the cell count, and the lexicon entry count - * are each bounded by {@link ResourceLimits#MAX_ENTRIES}.

+ * are each bounded by {@link ResourceLimits#MAX_ENTRIES}. Lexicon CSV fields may be + * MeCab-quoted with {@code ""} escapes. An {@code unk.def} template must name a + * category {@code char.def} defined.

* *

Instances are immutable and safe to share between threads.

* @@ -78,6 +81,9 @@ public final class MecabDictionary { /** The {@code char.def} field value that turns a category flag on. */ private static final String FLAG_ON = "1"; + /** The {@code char.def} field value that turns a category flag off. */ + private static final String FLAG_OFF = "0"; + /** * One lexicon or unknown-word entry. * @@ -466,9 +472,9 @@ private CategoryTable build(Map categories) throws IOException } /** - * Resolves a mapped category name against the defined categories, so a mapping to - * a name the {@code char.def} category section never defined fails at load with - * the offending code point instead of falling back silently at lookup time. + * Resolves a mapped category name against the defined categories. A mapping to a + * name the {@code char.def} category section never defined fails at load and names + * the offending code point. * * @param name The category name a mapping line gave. * @param categories The defined categories, keyed by name. @@ -530,7 +536,11 @@ private MecabDictionary(DoubleArrayLexicon lexicon, this.rightSize = rightSize; this.categoryTable = categoryTable; this.defaultCategory = categories.get(DEFAULT_CATEGORY); - this.unknownEntries = unknownEntries; + final Map> copy = new HashMap<>(unknownEntries.size()); + for (final Map.Entry> entry : unknownEntries.entrySet()) { + copy.put(entry.getKey(), List.copyOf(entry.getValue())); + } + this.unknownEntries = Map.copyOf(copy); } /** @@ -568,69 +578,86 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc } // The connection matrix is read first because its dimensions are what every // lexicon entry's context ids have to be inside of. - final List matrixLines = readLines(directory.resolve(MATRIX_DEF), charset); - final String headerLine = StringUtil.trimUnicodeWhitespace(matrixLines.get(0)); - if (headerLine.isEmpty()) { - throw new IOException("empty " + MATRIX_DEF + " under " + directory); - } - final String[] header = splitWhitespace(headerLine); - if (header.length != 2) { - throw new IOException("malformed " + MATRIX_DEF + " header: " + headerLine); - } - final int leftSize = parseInt(header[0], MATRIX_DEF, 1); - final int rightSize = parseInt(header[1], MATRIX_DEF, 1); - if (leftSize < 1 || rightSize < 1) { - throw new IOException(MATRIX_DEF + " dimensions must be positive, got " - + leftSize + " " + rightSize); - } - if (leftSize > ResourceLimits.MAX_ENTRIES - || rightSize > ResourceLimits.MAX_ENTRIES) { - throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize - + " exceed safe limit of " + ResourceLimits.MAX_ENTRIES); - } - final long cells = (long) leftSize * rightSize; - if (cells > Integer.MAX_VALUE) { - throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize - + " overflow the addressable connection matrix"); - } - if (cells > ResourceLimits.MAX_ENTRIES) { - throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize - + " exceed safe limit of " + ResourceLimits.MAX_ENTRIES); - } - final int cellCount = (int) cells; - final short[] costs = new short[cellCount]; - // leftSize bounds right-context ids and rightSize bounds left-context ids, matching - // MeCab's connector.h layout (the names read transposed against the id names). - final BitSet filled = new BitSet(cellCount); - for (int i = 1; i < matrixLines.size(); i++) { - final String line = StringUtil.trimUnicodeWhitespace(matrixLines.get(i)); - if (line.isEmpty()) { - continue; - } - final String[] fields = splitWhitespace(line); - if (fields.length != 3) { - throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1)); - } - final int right = parseInt(fields[0], MATRIX_DEF, i + 1); - final int left = parseInt(fields[1], MATRIX_DEF, i + 1); - if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { - throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) - + ": context ids " + right + " " + left - + " are outside the declared dimensions " + leftSize + " " + rightSize); - } - final int cost = parseInt(fields[2], MATRIX_DEF, i + 1); - if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { - throw new IOException("malformed " + MATRIX_DEF + " line " + (i + 1) - + ": connection cost " + cost + " is outside the 16-bit range the" - + " format defines"); - } - final int index = right * rightSize + left; - costs[index] = (short) cost; - filled.set(index); - } - if (filled.cardinality() != cellCount) { - throw new IOException(MATRIX_DEF + " declares " + leftSize + " x " + rightSize - + " connection costs but only " + filled.cardinality() + " pairs are listed"); + final Path matrixFile = directory.resolve(MATRIX_DEF); + if (!Files.exists(matrixFile)) { + throw new IOException("required dictionary file is missing: " + matrixFile); + } + final int leftSize; + final int rightSize; + final short[] costs; + final int cellCount; + try (BufferedReader reader = Files.newBufferedReader(matrixFile, charset)) { + final String rawHeader = reader.readLine(); + if (rawHeader == null) { + throw new IOException("empty " + MATRIX_DEF + " under " + directory); + } + final String headerLine = StringUtil.trimUnicodeWhitespace(rawHeader); + if (headerLine.isEmpty()) { + throw new IOException("empty " + MATRIX_DEF + " under " + directory); + } + final String[] header = splitWhitespace(headerLine); + if (header.length != 2) { + throw new IOException("malformed " + MATRIX_DEF + " header: " + headerLine); + } + leftSize = parseInt(header[0], MATRIX_DEF, 1); + rightSize = parseInt(header[1], MATRIX_DEF, 1); + if (leftSize < 1 || rightSize < 1) { + throw new IOException(MATRIX_DEF + " dimensions must be positive, got " + + leftSize + " " + rightSize); + } + if (leftSize > ResourceLimits.MAX_ENTRIES + || rightSize > ResourceLimits.MAX_ENTRIES) { + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + + " exceed safe limit of " + ResourceLimits.MAX_ENTRIES); + } + final long cells = (long) leftSize * rightSize; + if (cells > Integer.MAX_VALUE) { + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + + " overflow the addressable connection matrix"); + } + if (cells > ResourceLimits.MAX_ENTRIES) { + throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + + " exceed safe limit of " + ResourceLimits.MAX_ENTRIES); + } + cellCount = (int) cells; + costs = new short[cellCount]; + // leftSize bounds right-context ids and rightSize bounds left-context ids, matching + // MeCab's connector.h layout (the names read transposed against the id names). + final BitSet filled = new BitSet(cellCount); + int lineNumber = 1; + String raw; + while ((raw = reader.readLine()) != null) { + lineNumber++; + final String line = StringUtil.trimUnicodeWhitespace(raw); + if (line.isEmpty()) { + continue; + } + final String[] fields = splitWhitespace(line); + if (fields.length != 3) { + throw new IOException("malformed " + MATRIX_DEF + " line " + lineNumber); + } + final int right = parseInt(fields[0], MATRIX_DEF, lineNumber); + final int left = parseInt(fields[1], MATRIX_DEF, lineNumber); + if (right < 0 || right >= leftSize || left < 0 || left >= rightSize) { + throw new IOException("malformed " + MATRIX_DEF + " line " + lineNumber + + ": context ids " + right + " " + left + + " are outside the declared dimensions " + leftSize + " " + rightSize); + } + final int cost = parseInt(fields[2], MATRIX_DEF, lineNumber); + if (cost < Short.MIN_VALUE || cost > Short.MAX_VALUE) { + throw new IOException("malformed " + MATRIX_DEF + " line " + lineNumber + + ": connection cost " + cost + " is outside the 16-bit range the" + + " format defines"); + } + final int index = right * rightSize + left; + costs[index] = (short) cost; + filled.set(index); + } + if (filled.cardinality() != cellCount) { + throw new IOException(MATRIX_DEF + " declares " + leftSize + " x " + rightSize + + " connection costs but only " + filled.cardinality() + + " pairs are listed"); + } } final Map> lexicon = new HashMap<>(); @@ -654,8 +681,14 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc readCharacterDefinition(directory.resolve(CHAR_DEF), charset, categories, categoryTable); final Map> unknown = new HashMap<>(); - readLexicon(directory.resolve(UNK_DEF), charset, unknown, leftSize, rightSize, - new int[] {0}); + final Path unkFile = directory.resolve(UNK_DEF); + readLexicon(unkFile, charset, unknown, leftSize, rightSize, new int[] {0}); + for (final String category : unknown.keySet()) { + if (!categories.containsKey(category)) { + throw new IOException( + UNK_DEF + " names the undefined category " + category + ": " + unkFile); + } + } return new MecabDictionary(DoubleArrayLexicon.build(lexicon), costs, rightSize, categories, categoryTable.build(categories), unknown); @@ -678,41 +711,47 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc private static void readLexicon(Path file, Charset charset, Map> target, int leftSize, int rightSize, int[] entryCount) throws IOException { + if (!Files.exists(file)) { + throw new IOException("required dictionary file is missing: " + file); + } int lineNumber = 0; - for (final String line : readLines(file, charset)) { - lineNumber++; - if (line.isEmpty()) { - continue; - } - final List fields = splitCsv(line); - if (fields.size() < 4) { - throw new IOException("malformed entry at " + file + " line " + lineNumber); - } - final String surface = fields.get(0); - if (surface.isEmpty()) { - continue; - } - final int leftId = parseInt(fields.get(1), file.toString(), lineNumber); - final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); - if (leftId < 0 || leftId >= rightSize) { - throw new IOException("malformed entry at " + file + " line " + lineNumber - + ": left context id " + leftId + " is outside the " + MATRIX_DEF + " dimensions " - + leftSize + " " + rightSize); - } - if (rightId < 0 || rightId >= leftSize) { - throw new IOException("malformed entry at " + file + " line " + lineNumber - + ": right context id " + rightId + " is outside the " + MATRIX_DEF + " dimensions " - + leftSize + " " + rightSize); - } - if (entryCount[0] >= ResourceLimits.MAX_ENTRIES) { - throw new IOException("lexicon entry count exceeds safe limit of " - + ResourceLimits.MAX_ENTRIES); + try (BufferedReader reader = Files.newBufferedReader(file, charset)) { + String line; + while ((line = reader.readLine()) != null) { + lineNumber++; + if (line.isEmpty()) { + continue; + } + final List fields = splitCsv(line); + if (fields.size() < 4) { + throw new IOException("malformed entry at " + file + " line " + lineNumber); + } + final String surface = fields.get(0); + if (surface.isEmpty()) { + continue; + } + final int leftId = parseInt(fields.get(1), file.toString(), lineNumber); + final int rightId = parseInt(fields.get(2), file.toString(), lineNumber); + if (leftId < 0 || leftId >= rightSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": left context id " + leftId + " is outside the " + MATRIX_DEF + + " dimensions " + leftSize + " " + rightSize); + } + if (rightId < 0 || rightId >= leftSize) { + throw new IOException("malformed entry at " + file + " line " + lineNumber + + ": right context id " + rightId + " is outside the " + MATRIX_DEF + + " dimensions " + leftSize + " " + rightSize); + } + if (entryCount[0] >= ResourceLimits.MAX_ENTRIES) { + throw new IOException("lexicon entry count exceeds safe limit of " + + ResourceLimits.MAX_ENTRIES); + } + entryCount[0]++; + final WordEntry entry = new WordEntry(leftId, rightId, + parseInt(fields.get(3), file.toString(), lineNumber), + List.copyOf(fields.subList(4, fields.size()))); + target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); } - entryCount[0]++; - final WordEntry entry = new WordEntry(leftId, rightId, - parseInt(fields.get(3), file.toString(), lineNumber), - List.copyOf(fields.subList(4, fields.size()))); - target.computeIfAbsent(surface, key -> new ArrayList<>(1)).add(entry); } } @@ -731,45 +770,65 @@ private static void readLexicon(Path file, Charset charset, private static void readCharacterDefinition(Path file, Charset charset, Map categories, CategoryTableBuilder categoryTable) throws IOException { + if (!Files.exists(file)) { + throw new IOException("required dictionary file is missing: " + file); + } int lineNumber = 0; - for (final String raw : readLines(file, charset)) { - lineNumber++; - final String line = StringUtil.trimUnicodeWhitespace(stripComment(raw)); - if (line.isEmpty()) { - continue; - } - final String[] fields = splitWhitespace(line); - if (fields[0].regionMatches(true, 0, HEX_PREFIX, 0, HEX_PREFIX.length())) { - final int rangeSeparator = fields[0].indexOf(RANGE_SEPARATOR); - final int from; - final int to; - if (rangeSeparator >= 0) { - from = parseCodePoint(fields[0].substring(0, rangeSeparator), file, lineNumber); - to = parseCodePoint( - fields[0].substring(rangeSeparator + RANGE_SEPARATOR.length()), file, lineNumber); - } else { - from = parseCodePoint(fields[0], file, lineNumber); - to = from; - } - if (fields.length < 2) { - throw new IOException("mapping without category at " + file + " line " + lineNumber); - } - if (from > to) { - throw new IOException("code point range descends at " + file + " line " - + lineNumber); + try (BufferedReader reader = Files.newBufferedReader(file, charset)) { + String raw; + while ((raw = reader.readLine()) != null) { + lineNumber++; + final String line = StringUtil.trimUnicodeWhitespace(stripComment(raw)); + if (line.isEmpty()) { + continue; } - categoryTable.map(from, to, fields[1]); - } else { - if (fields.length < 4) { - throw new IOException("malformed category at " + file + " line " + lineNumber); + final String[] fields = splitWhitespace(line); + if (fields[0].regionMatches(true, 0, HEX_PREFIX, 0, HEX_PREFIX.length())) { + final int rangeSeparator = fields[0].indexOf(RANGE_SEPARATOR); + final int from; + final int to; + if (rangeSeparator >= 0) { + from = parseCodePoint(fields[0].substring(0, rangeSeparator), file, + lineNumber); + to = parseCodePoint( + fields[0].substring(rangeSeparator + RANGE_SEPARATOR.length()), file, + lineNumber); + } else { + from = parseCodePoint(fields[0], file, lineNumber); + to = from; + } + if (fields.length < 2) { + throw new IOException( + "mapping without category at " + file + " line " + lineNumber); + } + if (from > to) { + throw new IOException("code point range descends at " + file + " line " + + lineNumber); + } + categoryTable.map(from, to, fields[1]); + } else { + if (fields.length < 4) { + throw new IOException( + "malformed category at " + file + " line " + lineNumber); + } + if (!isFlag(fields[1]) || !isFlag(fields[2])) { + throw new IOException( + "malformed category flag at " + file + " line " + lineNumber); + } + final int length = parseInt(fields[3], file.toString(), lineNumber); + if (length < 0) { + throw new IOException( + "category LENGTH must not be negative at " + file + " line " + + lineNumber); + } + categories.put(fields[0], new Category(fields[0], + FLAG_ON.equals(fields[1]), FLAG_ON.equals(fields[2]), length)); } - categories.put(fields[0], new Category(fields[0], - FLAG_ON.equals(fields[1]), FLAG_ON.equals(fields[2]), - parseInt(fields[3], file.toString(), lineNumber))); } } if (!categories.containsKey(DEFAULT_CATEGORY)) { - throw new IOException(CHAR_DEF + " defines no " + DEFAULT_CATEGORY + " category: " + file); + throw new IOException( + CHAR_DEF + " defines no " + DEFAULT_CATEGORY + " category: " + file); } } @@ -821,35 +880,6 @@ List unknownEntries(String category) { return unknownEntries.get(category); } - /** - * Reads a required dictionary file into its lines, accepting LF and CRLF endings. - * - * @param file The file to read. - * @param charset The encoding to decode with. - * @return The lines without their line terminators. Never {@code null} and never - * empty: an empty file reads as one empty line. - * @throws IOException Thrown if the file is missing or reading fails. - */ - private static List readLines(Path file, Charset charset) throws IOException { - if (!Files.exists(file)) { - throw new IOException("required dictionary file is missing: " + file); - } - final String content = new String(Files.readAllBytes(file), charset); - final List lines = new ArrayList<>(); - int start = 0; - for (int i = 0; i <= content.length(); i++) { - if (i == content.length() || content.charAt(i) == '\n') { - int end = i; - if (end > start && content.charAt(end - 1) == '\r') { - end--; - } - lines.add(content.substring(start, end)); - start = i + 1; - } - } - return lines; - } - /** * Removes a trailing {@code #} comment from a {@code char.def} line. * @@ -863,21 +893,50 @@ private static String stripComment(String line) { } /** - * Splits a lexicon line at every comma. Surfaces containing commas are not - * representable, matching the plain-text lexicon format. + * Reports whether a {@code char.def} category flag field is exactly {@code 0} or + * {@code 1}. + * + * @param field The flag field text. + * @return {@code true} when the field is a recognized flag value. + */ + private static boolean isFlag(String field) { + return FLAG_ON.equals(field) || FLAG_OFF.equals(field); + } + + /** + * Splits a lexicon line on commas, honoring MeCab-style {@code "..."} quoting with + * {@code ""} escapes inside a quoted field. * * @param line The line to split. * @return The fields in order, empty fields included. Never {@code null}. */ private static List splitCsv(String line) { final List fields = new ArrayList<>(); - int start = 0; - for (int i = 0; i <= line.length(); i++) { - if (i == line.length() || line.charAt(i) == ',') { - fields.add(line.substring(start, i)); - start = i + 1; + final StringBuilder field = new StringBuilder(); + boolean inQuotes = false; + for (int i = 0; i < line.length(); i++) { + final char c = line.charAt(i); + if (inQuotes) { + if (c == '"') { + if (i + 1 < line.length() && line.charAt(i + 1) == '"') { + field.append('"'); + i++; + } else { + inQuotes = false; + } + } else { + field.append(c); + } + } else if (c == '"') { + inQuotes = true; + } else if (c == ',') { + fields.add(field.toString()); + field.setLength(0); + } else { + field.append(c); } } + fields.add(field.toString()); return fields; } diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java index 060a0a53d2..9260ad7698 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/UnigramSegmenter.java @@ -17,13 +17,16 @@ package opennlp.tools.tokenize.lattice; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,10 +58,53 @@ public class UnigramSegmenter implements Tokenizer { private final WordTrie trie; - /** A minimal trie over words with their log-probabilities. */ + /** + * One immutable trie node: children are a sorted character array with a parallel + * node array, found by binary search, so a descent never boxes a {@link Character}. + */ private static final class WordTrie { - private final Map children = new HashMap<>(); + + private final char[] keys; + private final WordTrie[] nodes; + private final double logProbability; + + private WordTrie(char[] keys, WordTrie[] nodes, double logProbability) { + this.keys = keys; + this.nodes = nodes; + this.logProbability = logProbability; + } + + /** + * Descends one character. + * + * @param c The next surface character. + * @return The child node, or {@code null} when no surface continues with {@code c}. + */ + private WordTrie child(char c) { + final int index = Arrays.binarySearch(keys, c); + return index >= 0 ? nodes[index] : null; + } + } + + /** One mutable trie node during construction, copied into a {@link WordTrie}. */ + private static final class WordTrieBuilder { + + private final Map children = new HashMap<>(); private double logProbability = Double.NaN; + + private WordTrie freeze() { + final char[] keys = new char[children.size()]; + int i = 0; + for (final Character key : children.keySet()) { + keys[i++] = key; + } + Arrays.sort(keys); + final WordTrie[] nodes = new WordTrie[keys.length]; + for (int k = 0; k < keys.length; k++) { + nodes[k] = children.get(keys[k]).freeze(); + } + return new WordTrie(keys, nodes, logProbability); + } } private UnigramSegmenter(WordTrie trie, double unknownLogProbability) { @@ -119,16 +165,13 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) } final Map counts = new HashMap<>(); long total = 0; - final String content = new String(lexiconStream.readAllBytes(), charset); - int lineStart = 0; + final BufferedReader reader = + new BufferedReader(new InputStreamReader(lexiconStream, charset)); int lineNumber = 0; - for (int i = 0; i <= content.length(); i++) { - if (i < content.length() && content.charAt(i) != '\n') { - continue; - } + String raw; + while ((raw = reader.readLine()) != null) { lineNumber++; - final String line = StringUtil.trimUnicodeWhitespace(content.substring(lineStart, i)); - lineStart = i + 1; + final String line = StringUtil.trimUnicodeWhitespace(raw); if (line.isEmpty()) { continue; } @@ -161,21 +204,20 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) throw new IOException("the lexicon lists no words"); } - final WordTrie root = new WordTrie(); + final WordTrieBuilder root = new WordTrieBuilder(); final double logTotal = Math.log(total); for (final Map.Entry entry : counts.entrySet()) { - WordTrie node = root; + WordTrieBuilder node = root; final String word = entry.getKey(); for (int c = 0; c < word.length(); c++) { - node = node.children.computeIfAbsent(word.charAt(c), - key -> new WordTrie()); + node = node.children.computeIfAbsent(word.charAt(c), key -> new WordTrieBuilder()); } node.logProbability = Math.log(entry.getValue()) - logTotal; } // Charge an unlisted character half of one count out of the total, which makes it // rarer than any listed word: every listed count is at least one. final double unknown = Math.log(0.5) - logTotal; - return new UnigramSegmenter(root, unknown); + return new UnigramSegmenter(root.freeze(), unknown); } /** @@ -187,12 +229,7 @@ public static UnigramSegmenter load(InputStream lexiconStream, Charset charset) */ @Override public String[] tokenize(String text) { - final Span[] spans = tokenizePos(text); - final String[] tokens = new String[spans.length]; - for (int i = 0; i < tokens.length; i++) { - tokens[i] = text.substring(spans[i].getStart(), spans[i].getEnd()); - } - return tokens; + return Span.spansToStrings(tokenizePos(text), text); } /** @@ -256,7 +293,7 @@ private void decode(String text, int from, int to, List spans) { } WordTrie node = trie; for (int j = from + i; j < to; j++) { - node = node.children.get(text.charAt(j)); + node = node.child(text.charAt(j)); if (node == null) { break; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 76212b1b5d..6974062e2c 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -17,6 +17,7 @@ package opennlp.tools.tokenize.lattice; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -26,18 +27,26 @@ import java.nio.file.StandardCopyOption; import java.util.zip.GZIPInputStream; +import opennlp.tools.util.model.UncloseableInputStream; + /** * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the * dictionary is acquired by the user at install time and never ships with this library. * The archive location is user-supplied; no dictionary data is bundled and no download * location is built in. * - *

The installer reads gzip-compressed tar archives, the format the common - * distributions use, and extracts only the dictionary payload: the {@code *.csv} - * lexicon files and {@code *.def} definition files that a {@link MecabDictionary} - * reads, plus the {@code dicrc} configuration file distributions ship alongside them. - * Entries are flattened to their base names, which also means no archive path can - * escape the target directory.

+ *

The installer reads gzip-compressed ustar archives (POSIX.1-1988), the format the + * common distributions use. GNU long-name ({@code L}) and PAX ({@code x}/{@code g}) + * headers are not supported; entry names must fit the 100-byte ustar name field. It + * extracts only the dictionary payload: the {@code *.csv} lexicon files and + * {@code *.def} definition files that a {@link MecabDictionary} reads, plus the + * {@code dicrc} configuration file distributions ship alongside them. Entries are + * flattened to their base names, which also means no archive path can escape the target + * directory.

+ * + *

Extraction is bounded: each entry's declared size, the total bytes written, the + * number of extracted dictionary files, and the gzip expansion ratio each have an + * explicit ceiling so a crafted archive cannot fill the disk.

* * @since 3.0.0 */ @@ -49,6 +58,21 @@ public final class MecabDictionaryInstaller { private static final int TAR_SIZE_LENGTH = 12; private static final int TAR_TYPE_OFFSET = 156; + /** Inclusive ceiling on one tar entry's declared size, in bytes. */ + static final long MAX_ENTRY_BYTES = 512L * 1024 * 1024; + + /** Inclusive ceiling on the sum of extracted dictionary file sizes, in bytes. */ + static final long MAX_TOTAL_EXTRACTED_BYTES = 2L * 1024 * 1024 * 1024; + + /** Inclusive ceiling on the number of dictionary files extracted from one archive. */ + static final int MAX_EXTRACTED_ENTRIES = 10_000; + + /** + * Inclusive ceiling on decompressed bytes per compressed byte while reading the + * gzip wrapper; higher expansion fails before the payload is written. + */ + static final int MAX_GZIP_EXPANSION_RATIO = 100; + private MecabDictionaryInstaller() { // This class exposes only static methods and is never instantiated. } @@ -56,13 +80,13 @@ private MecabDictionaryInstaller() { /** * Downloads a dictionary archive and unpacks it. * - * @param archive The archive location, a gzip-compressed tar. Must not be + * @param archive The archive location, a gzip-compressed ustar tar. Must not be * {@code null}. * @param targetDirectory The directory to unpack into; created when absent. Must not * be {@code null}. * @return The number of dictionary files extracted. - * @throws IOException Thrown if fetching, reading, or writing fails, or the archive - * contains no dictionary file. + * @throws IOException Thrown if fetching, reading, or writing fails, the archive + * contains no dictionary file, or an extraction budget is exceeded. * @throws IllegalArgumentException Thrown if a parameter is {@code null} or * {@code archive} is not an absolute URI. */ @@ -79,19 +103,41 @@ public static int install(URI archive, Path targetDirectory) throws IOException } /** - * Unpacks a dictionary archive stream. + * Unpacks a dictionary archive stream under the production extraction budgets. * - * @param archiveStream The gzip-compressed tar content. Must not be {@code null}. - * Not closed. + * @param archiveStream The gzip-compressed ustar tar content. Must not be + * {@code null}. Not closed. * @param targetDirectory The directory to unpack into; created when absent. Must not * be {@code null}. * @return The number of dictionary files extracted. - * @throws IOException Thrown if reading or writing fails, or the archive contains no - * dictionary file. + * @throws IOException Thrown if reading or writing fails, the archive contains no + * dictionary file, or an extraction budget is exceeded. * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static int extract(InputStream archiveStream, Path targetDirectory) throws IOException { + return extract(archiveStream, targetDirectory, MAX_ENTRY_BYTES, + MAX_TOTAL_EXTRACTED_BYTES, MAX_EXTRACTED_ENTRIES, MAX_GZIP_EXPANSION_RATIO); + } + + /** + * Unpacks a dictionary archive stream under caller-supplied budgets. + * + * @param archiveStream The gzip-compressed ustar tar content. Must not be + * {@code null}. Not closed. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @param maxEntryBytes Inclusive ceiling on one entry's declared size. + * @param maxTotalBytes Inclusive ceiling on total extracted bytes. + * @param maxEntries Inclusive ceiling on extracted dictionary file count. + * @param maxGzipRatio Inclusive ceiling on decompressed bytes per compressed byte. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if reading or writing fails, the archive contains no + * dictionary file, or a budget is exceeded. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + static int extract(InputStream archiveStream, Path targetDirectory, long maxEntryBytes, + long maxTotalBytes, int maxEntries, int maxGzipRatio) throws IOException { if (archiveStream == null) { throw new IllegalArgumentException("archiveStream must not be null"); } @@ -99,35 +145,54 @@ public static int extract(InputStream archiveStream, Path targetDirectory) throw new IllegalArgumentException("targetDirectory must not be null"); } Files.createDirectories(targetDirectory); - final InputStream tar = new GZIPInputStream(archiveStream); - final byte[] header = new byte[TAR_BLOCK]; - int extracted = 0; - while (readBlock(tar, header)) { - if (isEndBlock(header)) { - break; - } - final String name = headerName(header); - final long size = headerSize(header); - final char type = (char) header[TAR_TYPE_OFFSET]; - final String baseName = baseName(name); - final boolean wanted = (type == '0' || type == 0) - && (baseName.endsWith(".csv") || baseName.endsWith(".def") - || "dicrc".equals(baseName)); - if (wanted) { - final Path file = targetDirectory.resolve(baseName); - try (InputStream entry = boundedStream(tar, size)) { - Files.copy(entry, file, StandardCopyOption.REPLACE_EXISTING); + final CountingInputStream compressed = new CountingInputStream(archiveStream); + try (GZIPInputStream gzip = new GZIPInputStream( + new UncloseableInputStream(compressed))) { + final BudgetedInputStream tar = + new BudgetedInputStream(gzip, compressed, maxGzipRatio); + final byte[] header = new byte[TAR_BLOCK]; + int extracted = 0; + long totalExtracted = 0; + while (readBlock(tar, header)) { + if (isEndBlock(header)) { + break; + } + final String name = headerName(header); + final long size = headerSize(header); + if (size > maxEntryBytes) { + throw new IOException( + "tar entry size exceeds safe limit of " + maxEntryBytes); + } + final char type = (char) header[TAR_TYPE_OFFSET]; + final String baseName = baseName(name); + final boolean wanted = (type == '0' || type == 0) + && (baseName.endsWith(".csv") || baseName.endsWith(".def") + || "dicrc".equals(baseName)); + if (wanted) { + if (extracted >= maxEntries) { + throw new IOException( + "extracted entry count exceeds safe limit of " + maxEntries); + } + if (totalExtracted + size > maxTotalBytes) { + throw new IOException( + "extracted archive size exceeds safe limit of " + maxTotalBytes); + } + final Path file = targetDirectory.resolve(baseName); + try (InputStream entry = boundedStream(tar, size)) { + Files.copy(entry, file, StandardCopyOption.REPLACE_EXISTING); + } + extracted++; + totalExtracted += size; + skip(tar, padding(size)); + } else { + skip(tar, size + padding(size)); } - extracted++; - skip(tar, padding(size)); - } else { - skip(tar, size + padding(size)); } + if (extracted == 0) { + throw new IOException("the archive contains no dictionary file"); + } + return extracted; } - if (extracted == 0) { - throw new IOException("the archive contains no dictionary file"); - } - return extracted; } /** @@ -286,4 +351,84 @@ public int read(byte[] buffer, int offset, int length) throws IOException { } }; } + + /** + * Counts bytes read from a delegate stream. + */ + private static final class CountingInputStream extends FilterInputStream { + + private long count; + + private CountingInputStream(InputStream in) { + super(in); + } + + private long count() { + return count; + } + + @Override + public int read() throws IOException { + final int b = super.read(); + if (b >= 0) { + count++; + } + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + final int read = super.read(buffer, offset, length); + if (read > 0) { + count += read; + } + return read; + } + } + + /** + * Counts decompressed bytes and rejects a gzip expansion above the supplied ratio. + */ + private static final class BudgetedInputStream extends FilterInputStream { + + private final CountingInputStream compressed; + private final int maxGzipRatio; + private long decompressed; + + private BudgetedInputStream(InputStream in, CountingInputStream compressed, + int maxGzipRatio) { + super(in); + this.compressed = compressed; + this.maxGzipRatio = maxGzipRatio; + } + + @Override + public int read() throws IOException { + final int b = super.read(); + if (b >= 0) { + decompressed++; + checkRatio(); + } + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + final int read = super.read(buffer, offset, length); + if (read > 0) { + decompressed += read; + checkRatio(); + } + return read; + } + + private void checkRatio() throws IOException { + final long compressedBytes = compressed.count(); + if (compressedBytes > 0 + && decompressed > (long) maxGzipRatio * compressedBytes) { + throw new IOException( + "gzip expansion ratio exceeds safe limit of " + maxGzipRatio); + } + } + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index b5e6c0722a..324bf9e058 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -403,7 +403,10 @@ void testEmptyMatrixDefFailsLoud(@TempDir Path broken) throws IOException { DEFAULT_CATEGORY_LINE + "\n0x0110..0x0100 LATIN\n", DEFAULT_CATEGORY_LINE + "\n0x110000 LATIN\n", DEFAULT_CATEGORY_LINE + "\n0xZZ LATIN\n", - "DEFAULT 0 1\n"}) + "DEFAULT 0 1\n", + "DEFAULT 2 1 0\n", + "DEFAULT 0 true 0\n", + "DEFAULT 0 1 -1\n"}) void testMalformedCharDefFailsLoud(String charDef, @TempDir Path broken) throws IOException { write(broken, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); @@ -413,6 +416,42 @@ void testMalformedCharDefFailsLoud(String charDef, @TempDir Path broken) Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); } + /** + * Verifies that a MeCab-style quoted CSV field may contain a comma, with {@code ""} + * escaping a literal quote, and that the loaded features keep both intact. + */ + @Test + void testQuotedCsvFieldWithCommaLoads(@TempDir Path quoted) throws IOException { + write(quoted, LEXICON_CSV, + "\u6771,0,0,3000,\"noun,common\",\"say \"\"hi\"\"\"\n"); + write(quoted, MATRIX_DEF, UNIT_MATRIX); + write(quoted, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n"); + write(quoted, UNK_DEF, DEFAULT_UNKNOWN_TEMPLATE + "\n"); + + final List morphemes = + new LatticeTokenizer(MecabDictionary.load(quoted)).analyze("\u6771"); + Assertions.assertEquals(1, morphemes.size()); + Assertions.assertEquals(List.of("noun,common", "say \"hi\""), + morphemes.get(0).features()); + } + + /** + * Verifies that an {@code unk.def} template naming a category {@code char.def} never + * defined fails at load with {@link IOException}. + */ + @Test + void testUnkDefUndefinedCategoryFailsLoud(@TempDir Path ghost) throws IOException { + write(ghost, LEXICON_CSV, "\u6771,0,0,3000,noun\n"); + write(ghost, MATRIX_DEF, UNIT_MATRIX); + write(ghost, CHAR_DEF, DEFAULT_CATEGORY_LINE + "\n"); + write(ghost, UNK_DEF, "GHOST,0,0,8000,noun\n"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(ghost)); + Assertions.assertEquals("unk.def names the undefined category GHOST: " + + ghost.resolve(UNK_DEF), e.getMessage()); + } + /** * Writes a miniature dictionary whose {@code char.def} maps a supplementary plane * range, the shape a UniDic-style distribution uses for the CJK extension blocks. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index fb5c6a701f..5141025d8b 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -81,6 +82,76 @@ void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) new ByteArrayInputStream(archive), target)); } + /** + * Verifies that a tar entry whose declared size is above the per-entry ceiling is + * rejected before any payload is written. The fixture stores only the oversized + * header so the test does not allocate the declared size. + */ + @Test + void testOversizedEntryFailsLoud(@TempDir Path target) throws IOException { + final long limit = 64; + final byte[] archive = TarGzArchives.gzippedTar( + TarGzArchives.Entry.withDeclaredSize("huge.csv", new byte[0], limit + 1)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive), + target, limit, 1024, 16, 100)); + Assertions.assertEquals("tar entry size exceeds safe limit of " + limit, + e.getMessage()); + } + + /** + * Verifies that extracting dictionary files whose sizes sum above the total-bytes + * ceiling fails with {@link IOException}. + */ + @Test + void testTotalExtractedBytesBudgetFailsLoud(@TempDir Path target) throws IOException { + final long limit = 30; + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { + {"a.csv", "01234567890123456789"}, + {"b.csv", "01234567890123456789"}}); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive), + target, 1024, limit, 16, 100)); + Assertions.assertEquals("extracted archive size exceeds safe limit of " + limit, + e.getMessage()); + } + + /** + * Verifies that an archive with more dictionary files than the entry-count ceiling + * fails on the entry that would exceed it. + */ + @Test + void testExtractedEntryCountBudgetFailsLoud(@TempDir Path target) throws IOException { + final int limit = 2; + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { + {"a.csv", "a\n"}, + {"b.def", "b\n"}, + {"c.csv", "c\n"}}); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive), + target, 1024, 1024, limit, 100)); + Assertions.assertEquals("extracted entry count exceeds safe limit of " + limit, + e.getMessage()); + } + + /** + * Verifies that a highly compressible payload whose expansion exceeds the gzip + * ratio ceiling fails loud before the inflated content is kept. + */ + @Test + void testGzipExpansionRatioBudgetFailsLoud(@TempDir Path target) throws IOException { + final int ratio = 2; + final byte[] zeros = new byte[64 * 1024]; + Arrays.fill(zeros, (byte) 0); + final byte[] archive = TarGzArchives.gzippedTar( + TarGzArchives.Entry.of("zeros.csv", zeros)); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionaryInstaller.extract(new ByteArrayInputStream(archive), + target, zeros.length, zeros.length, 16, ratio)); + Assertions.assertEquals("gzip expansion ratio exceeds safe limit of " + ratio, + e.getMessage()); + } + @Test void testInvalidArguments(@TempDir Path target) { Assertions.assertThrows(IllegalArgumentException.class, diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java index 896115aba2..aaaffe1f01 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/TarGzArchives.java @@ -46,6 +46,56 @@ final class TarGzArchives { private TarGzArchives() { } + /** + * One archive entry: a path name, the bytes stored after the header, and the size + * field written into the header (which may differ from the stored content length so + * budget checks can be exercised without allocating the declared payload). + * + * @param name The entry name including any directory prefix. + * @param content The bytes written after the header; may be shorter than + * {@code declaredSize}. + * @param declaredSize The octal size field stored in the header. + */ + record Entry(String name, byte[] content, long declaredSize) { + + /** + * Builds an entry whose declared size matches its UTF-8 content length. + * + * @param name The entry name. + * @param content The entry text. + * @return The entry. Never {@code null}. + */ + static Entry of(String name, String content) { + final byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + return new Entry(name, bytes, bytes.length); + } + + /** + * Builds an entry whose declared size matches its content length. + * + * @param name The entry name. + * @param content The entry bytes. + * @return The entry. Never {@code null}. + */ + static Entry of(String name, byte[] content) { + return new Entry(name, content, content.length); + } + + /** + * Builds an entry whose header size field is set independently of the stored + * content, for oversized-entry budget tests. + * + * @param name The entry name. + * @param content The bytes stored after the header; typically empty for header-only + * oversized cases. + * @param declaredSize The size field written into the header. + * @return The entry. Never {@code null}. + */ + static Entry withDeclaredSize(String name, byte[] content, long declaredSize) { + return new Entry(name, content, declaredSize); + } + } + /** * Builds a gzip-compressed tar archive from name and content pairs, the layout a * dictionary distribution ships in. @@ -56,9 +106,24 @@ private TarGzArchives() { * @throws IOException Thrown if writing to the in-memory streams fails. */ static byte[] gzippedTar(String[][] entries) throws IOException { + final Entry[] typed = new Entry[entries.length]; + for (int i = 0; i < entries.length; i++) { + typed[i] = Entry.of(entries[i][0], entries[i][1]); + } + return gzippedTar(typed); + } + + /** + * Builds a gzip-compressed tar archive from typed entries. + * + * @param entries The entries to store. Must not be {@code null}. + * @return The compressed archive bytes. Never {@code null}. + * @throws IOException Thrown if writing to the in-memory streams fails. + */ + static byte[] gzippedTar(Entry... entries) throws IOException { final ByteArrayOutputStream tar = new ByteArrayOutputStream(); - for (final String[] entry : entries) { - tarEntry(tar, entry[0], entry[1].getBytes(StandardCharsets.UTF_8)); + for (final Entry entry : entries) { + tarEntry(tar, entry); } // Two zero blocks end a tar archive. tar.write(new byte[2 * BLOCK]); @@ -71,28 +136,30 @@ static byte[] gzippedTar(String[][] entries) throws IOException { /** * Appends one ustar file entry to a growing tar image: a 512-byte header block - * followed by the content padded to a block boundary. + * followed by the stored content padded to a block boundary of the declared size + * when content is present, or the header alone when the test supplies no payload. * * @param tar The tar image under construction. Must not be {@code null}. - * @param name The entry name including any directory prefix. Must not be - * {@code null}, empty, or longer than the 100-byte header name field. - * @param content The entry content bytes. Must not be {@code null}. + * @param entry The entry to append. Must not be {@code null}. * @throws IOException Thrown if writing to the in-memory stream fails. - * @throws IllegalArgumentException Thrown if {@code name} does not fit the header. + * @throws IllegalArgumentException Thrown if {@code name} does not fit the header or + * {@code declaredSize} is negative. */ - private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] content) - throws IOException { - final byte[] nameBytes = name.getBytes(StandardCharsets.UTF_8); + private static void tarEntry(ByteArrayOutputStream tar, Entry entry) throws IOException { + final byte[] nameBytes = entry.name().getBytes(StandardCharsets.UTF_8); if (nameBytes.length == 0 || nameBytes.length > NAME_LENGTH) { throw new IllegalArgumentException( "entry name must be 1.." + NAME_LENGTH + " bytes, got " + nameBytes.length); } + if (entry.declaredSize() < 0) { + throw new IllegalArgumentException("declaredSize must not be negative"); + } final byte[] header = new byte[BLOCK]; System.arraycopy(nameBytes, 0, header, 0, nameBytes.length); final byte[] mode = "0000644".getBytes(StandardCharsets.US_ASCII); System.arraycopy(mode, 0, header, MODE_OFFSET, mode.length); // Both numeric fields hold octal digits followed by one terminator byte. - final byte[] size = String.format("%0" + (SIZE_LENGTH - 1) + "o", content.length) + final byte[] size = String.format("%0" + (SIZE_LENGTH - 1) + "o", entry.declaredSize()) .getBytes(StandardCharsets.US_ASCII); System.arraycopy(size, 0, header, SIZE_OFFSET, size.length); header[TYPE_OFFSET] = REGULAR_FILE; @@ -110,8 +177,37 @@ private static void tarEntry(ByteArrayOutputStream tar, String name, byte[] cont header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 2] = 0; header[CHECKSUM_OFFSET + CHECKSUM_LENGTH - 1] = ' '; tar.write(header); - tar.write(content); - final int padding = (BLOCK - content.length % BLOCK) % BLOCK; + if (entry.declaredSize() == 0) { + return; + } + if (entry.content().length == 0) { + // Header-only oversized fixtures: the extractor rejects on the size field before + // reading a payload, so the declared bytes are not materialised here. + return; + } + tar.write(entry.content()); + final long missing = Math.max(0, entry.declaredSize() - entry.content().length); + if (missing > 0) { + writeZeros(tar, missing); + } + final int padding = (BLOCK - (int) (entry.declaredSize() % BLOCK)) % BLOCK; tar.write(new byte[padding]); } + + /** + * Writes {@code count} zero bytes to the stream. + * + * @param out The stream to write to. + * @param count The number of zero bytes. + * @throws IOException Thrown if writing fails. + */ + private static void writeZeros(ByteArrayOutputStream out, long count) throws IOException { + final byte[] zeros = new byte[8192]; + long remaining = count; + while (remaining > 0) { + final int chunk = (int) Math.min(zeros.length, remaining); + out.write(zeros, 0, chunk); + remaining -= chunk; + } + } } diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index f9f5466a46..1b376abc67 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -550,6 +550,9 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { matrix.def must list a cost for every declared cell, and matrix dimensions plus lexicon size are bounded by the shared ResourceLimits.MAX_ENTRIES limit. + Extraction caps per-entry size, total bytes, entry count, and gzip expansion. + Load also rejects an unk.def template for a category + char.def never defined, and accepts MeCab-quoted CSV fields. LatticeUsageExampleTest asserts the install-load-tokenize and lexicon flows shown here. From b81f34840426977134980c20afde773524386f75 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 10:53:05 -0400 Subject: [PATCH 17/24] OPENNLP-1894: Verify dictionary downloads by SHA-512 and add an opt-in URL catalog --- dev/README-mecab-dictionaries.md | 57 ++++-- dev/download-mecab-dictionary.sh | 69 ------- .../lattice/MecabDictionaryInstaller.java | 103 ++++++++++- .../opennlp/tools/util/DictionaryCatalog.java | 169 +++++++++++++++++ .../java/opennlp/tools/util/DownloadUtil.java | 170 +++++++++++++++++- .../tools/util/dictionary-catalog.properties | 57 ++++++ .../lattice/LatticeUsageExampleTest.java | 8 +- .../lattice/MecabDictionaryInstallerTest.java | 47 +++++ .../tools/util/DictionaryCatalogTest.java | 95 ++++++++++ .../opennlp/tools/util/DigestTestUtil.java | 45 +++++ .../tools/util/DownloadUtilFileTest.java | 100 +++++++++++ opennlp-docs/src/docbkx/tokenizer.xml | 7 +- 12 files changed, 826 insertions(+), 101 deletions(-) delete mode 100755 dev/download-mecab-dictionary.sh create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java create mode 100644 opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java diff --git a/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md index 6d1be9420f..5284e43560 100644 --- a/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -21,38 +21,57 @@ The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and K ## Known mecab-format dictionary projects -| Dictionary | Language | Archive to download | Encoding of the files | +| Catalog id | Dictionary | Language | Encoding | |---|---|---|---| -| IPADIC 2.7.0 | Japanese | `mecab-ipadic-2.7.0-20070801.tar.gz` from the MeCab project's SourceForge file area (`sourceforge.net/projects/mecab/files/mecab-ipadic/2.7.0-20070801/`) | EUC-JP | -| mecab-ko-dic 2.1.1 | Korean | `mecab-ko-dic-2.1.1-20180720.tar.gz` from the project's Bitbucket downloads page (`bitbucket.org/eunjeon/mecab-ko-dic/downloads/`) | UTF-8 | +| `mecab.ipadic` | IPADIC 2.7.0 | Japanese | EUC-JP | +| `mecab.ko-dic` | mecab-ko-dic 2.1.1 | Korean | UTF-8 | -Both archives are gzip-compressed tars, the format `MecabDictionaryInstaller` reads. The encoding column matters: `MecabDictionary.load(Path)` assumes UTF-8, and a dictionary in any other encoding is loaded with the two-argument overload, for example `MecabDictionary.load(dir, Charset.forName("EUC-JP"))` for IPADIC. +Pinned download URLs and SHA-512 digests for those ids live in +`opennlp/tools/util/dictionary-catalog.properties`. Both archives are +gzip-compressed ustar tars, the format `MecabDictionaryInstaller` reads. -## Step 1: download the archive +The installer extracts only the dictionary payload: the `*.csv` and `*.def` files a +`MecabDictionary` reads, plus the `dicrc` configuration file the distributions ship +alongside them. It flattens the entries into the target directory, and by the same +flattening makes it impossible for an archive path to escape that directory. The +returned count is the number of dictionary files extracted. -Either fetch the archive with any tool you like, or use the helper next to this file, which adds checksum verification: +## Option A: opt-in catalog install -``` -./download-mecab-dictionary.sh ipadic.tar.gz [expected-sha256] -``` +Catalog URLs stay inactive until you set `-Dopennlp.download.remote=true` (or the +equivalent system property in code). That flag is the explicit user action that +enables the built-in URLs; OpenNLP never fetches them by default. -On the first run without a checksum the script prints the SHA-256 it computed; record it and pass it on later runs so a changed or corrupted download fails loudly instead of being installed. +```java +import java.nio.file.Path; +import opennlp.tools.tokenize.lattice.MecabDictionaryInstaller; -## Step 2: unpack with the installer +// JVM flag: -Dopennlp.download.remote=true +int files = MecabDictionaryInstaller.installFromCatalog( + "mecab.ipadic", Path.of("ipadic")); +``` -The installer extracts only the dictionary payload: the `*.csv` and `*.def` files a `MecabDictionary` reads, plus the `dicrc` configuration file the distributions ship alongside them. It flattens the entries into the target directory, and by the same flattening makes it impossible for an archive path to escape that directory: +## Option B: your own URL and digest ```java +import java.net.URI; import java.nio.file.Path; import opennlp.tools.tokenize.lattice.MecabDictionaryInstaller; +String expectedSha512 = "..."; // the 128-hex SHA-512 of the archive int files = MecabDictionaryInstaller.install( - Path.of("ipadic.tar.gz").toUri(), Path.of("ipadic")); + URI.create("https://example.example/dict.tar.gz"), + Path.of("dict"), + expectedSha512); ``` -The returned count is the number of dictionary files extracted. A remote URI works in the same call if you prefer to skip step 1 entirely and let the installer stream the download. +A local `file:` URI may omit the digest: +`MecabDictionaryInstaller.install(localArchive.toUri(), targetDirectory)`. +Any other URI scheme requires the digest. + +## Load and tokenize -## Step 3: load and tokenize +`MecabDictionary.load(Path)` assumes UTF-8. IPADIC needs the two-argument overload: ```java import java.nio.charset.Charset; @@ -67,11 +86,15 @@ LatticeTokenizer tokenizer = new LatticeTokenizer(dictionary); String[] tokens = tokenizer.tokenize("\u6771\u4EAC\u90FD\u306B\u884C\u304F"); ``` -For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` is enough. Loaded dictionaries and tokenizers are immutable and safe to share between threads, so load once and reuse. +For a UTF-8 dictionary such as mecab-ko-dic, `MecabDictionary.load(Path.of("ko-dic"))` +is enough. Loaded dictionaries and tokenizers are immutable and safe to share between +threads, so load once and reuse. ## Chinese: the unigram segmenter needs only a frequency lexicon -`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use mecab dictionaries. It loads a plain text lexicon, one entry per line: the word, its count, and optionally a tag, separated by whitespace. Any word-frequency list you have the rights to use works: +`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use mecab dictionaries. It +loads a plain text lexicon, one entry per line: the word, its count, and optionally a +tag, separated by whitespace. Any word-frequency list you have the rights to use works: ```java import java.nio.file.Path; diff --git a/dev/download-mecab-dictionary.sh b/dev/download-mecab-dictionary.sh deleted file mode 100755 index 3b86ed09d6..0000000000 --- a/dev/download-mecab-dictionary.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Downloads a mecab-format dictionary archive for the lattice tokenizer. -# -# This script only fetches the archive and, when a checksum is given, verifies it. -# It does not unpack anything: unpacking is the job of -# opennlp.tools.tokenize.lattice.MecabDictionaryInstaller, which extracts exactly the -# files a MecabDictionary reads and nothing else. See README-mecab-dictionaries.md in -# this directory for the known dictionary projects, their encodings, and the Java -# steps that follow the download. -# -# The dictionary you download carries its own license. Nothing is bundled with -# Apache OpenNLP and no dictionary location is built in. - -set -euo pipefail - -usage() { - echo "usage: $0 [expected-sha256]" >&2 - echo "" >&2 - echo " archive-url where to fetch the gzip-compressed tar from" >&2 - echo " target-file where to store the downloaded archive" >&2 - echo " expected-sha256 optional checksum; when given, the download is verified" >&2 - echo " and a mismatch deletes the file and fails the script" >&2 - exit 2 -} - -[ $# -lt 2 ] && usage -url="$1" -target="$2" -expected="${3:-}" - -# Download to a temporary name first so an interrupted transfer never leaves a -# half-written file at the target path. -tmp="${target}.download" -echo "downloading ${url}" -curl --fail --location --retry 3 --output "${tmp}" "${url}" - -actual="$(sha256sum "${tmp}" | cut -d' ' -f1)" -if [ -n "${expected}" ]; then - if [ "${actual}" != "${expected}" ]; then - rm -f "${tmp}" - echo "checksum mismatch: expected ${expected}" >&2 - echo " but got ${actual}" >&2 - exit 1 - fi - echo "checksum verified: ${actual}" -else - echo "sha256 of the download: ${actual}" - echo "record this value and pass it as the third argument next time to verify" -fi - -mv "${tmp}" "${target}" -echo "stored ${target}" -echo "" -echo "next: unpack and load it from Java; see README-mecab-dictionaries.md" diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 6974062e2c..b662ca00b2 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -27,13 +27,16 @@ import java.nio.file.StandardCopyOption; import java.util.zip.GZIPInputStream; +import opennlp.tools.util.DictionaryCatalog; +import opennlp.tools.util.DownloadUtil; import opennlp.tools.util.model.UncloseableInputStream; /** * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the * dictionary is acquired by the user at install time and never ships with this library. - * The archive location is user-supplied; no dictionary data is bundled and no download - * location is built in. + * No dictionary data is bundled. Any non-{@code file:} archive is downloaded through + * {@link DownloadUtil#download(URI, Path, String)} and requires an expected SHA-512 + * digest. Built-in catalog URLs are opt-in via {@link #installFromCatalog(String, Path)}. * *

The installer reads gzip-compressed ustar archives (POSIX.1-1988), the format the * common distributions use. GNU long-name ({@code L}) and PAX ({@code x}/{@code g}) @@ -78,30 +81,112 @@ private MecabDictionaryInstaller() { } /** - * Downloads a dictionary archive and unpacks it. + * Unpacks a local {@code file:} archive URI. Any other scheme requires + * {@link #install(URI, Path, String)} with an expected SHA-512 digest. * * @param archive The archive location, a gzip-compressed ustar tar. Must not be * {@code null}. * @param targetDirectory The directory to unpack into; created when absent. Must not * be {@code null}. * @return The number of dictionary files extracted. - * @throws IOException Thrown if fetching, reading, or writing fails, the archive - * contains no dictionary file, or an extraction budget is exceeded. - * @throws IllegalArgumentException Thrown if a parameter is {@code null} or - * {@code archive} is not an absolute URI. + * @throws IOException Thrown if reading or writing fails, the archive contains no + * dictionary file, or an extraction budget is exceeded. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code archive} is not an absolute URI, or {@code archive} is not a + * {@code file:} URI. */ public static int install(URI archive, Path targetDirectory) throws IOException { + return install(archive, targetDirectory, null); + } + + /** + * Downloads a dictionary archive when needed, verifies its SHA-512 digest through + * {@link DownloadUtil#download(URI, Path, String)}, and unpacks it. A {@code file:} + * URI may omit the digest and is then opened without verification. + * + * @param archive The archive location, a gzip-compressed ustar tar. Must not be + * {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @param expectedSha512 The expected SHA-512 hex digest. Required for any + * non-{@code file:} URI; optional for {@code file:} URIs. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if fetching, verification, reading, or writing fails, + * the archive contains no dictionary file, or an extraction budget is exceeded. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code archive} is not an absolute URI, or a non-{@code file:} URI omits + * the digest. + */ + public static int install(URI archive, Path targetDirectory, String expectedSha512) + throws IOException { if (archive == null) { throw new IllegalArgumentException("archive must not be null"); } if (targetDirectory == null) { throw new IllegalArgumentException("targetDirectory must not be null"); } - try (InputStream in = archive.toURL().openStream()) { - return extract(in, targetDirectory); + if (!archive.isAbsolute()) { + throw new IllegalArgumentException("archive must be an absolute URI"); + } + if (expectedSha512 == null) { + if (!isLocalFile(archive)) { + throw new IllegalArgumentException("a non-file archive requires an expected " + + "SHA-512 digest; use install(URI, Path, String)"); + } + try (InputStream in = archive.toURL().openStream()) { + return extract(in, targetDirectory); + } + } + final Path downloaded = Files.createTempFile("mecab-dict-", ".tar.gz"); + try { + DownloadUtil.download(archive, downloaded, expectedSha512); + try (InputStream in = Files.newInputStream(downloaded)) { + return extract(in, targetDirectory); + } + } finally { + Files.deleteIfExists(downloaded); + } + } + + /** + * Downloads a dictionary named in {@link DictionaryCatalog} and unpacks it. Requires + * {@code -Dopennlp.download.remote=true}. + * + * @param dictionaryId The catalog id, for example {@code mecab.ipadic} or + * {@code mecab.ko-dic}. Must not be {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if the catalog entry is missing, remote downloads are + * disabled, or install fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int installFromCatalog(String dictionaryId, Path targetDirectory) + throws IOException { + if (dictionaryId == null) { + throw new IllegalArgumentException("dictionaryId must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); + } + final Path downloaded = Files.createTempFile("mecab-dict-", ".tar.gz"); + try { + DictionaryCatalog.loadDefault().download(dictionaryId, downloaded); + return install(downloaded.toUri(), targetDirectory); + } finally { + Files.deleteIfExists(downloaded); } } + /** + * {@return {@code true} when {@code archive} uses the {@code file} scheme} + * + * @param archive The absolute archive URI. + */ + private static boolean isLocalFile(URI archive) { + return "file".equalsIgnoreCase(archive.getScheme()); + } + /** * Unpacks a dictionary archive stream under the production extraction budgets. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java new file mode 100644 index 0000000000..5e32953346 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Properties; +import java.util.Set; + +/** + * Opt-in catalog of remote dictionary archives and companion files. The catalog + * ships URLs and SHA-512 digests only; it never bundles the data itself. Fetching + * an entry requires {@link DownloadUtil#REMOTE_DOWNLOAD_PROPERTY} to be + * {@code true}, so enabling a built-in URL is an explicit user action. + * + * @since 3.0.0 + */ +public final class DictionaryCatalog { + + private static final String DEFAULT_RESOURCE = + "opennlp/tools/util/dictionary-catalog.properties"; + + private final Properties properties; + + private DictionaryCatalog(Properties properties) { + this.properties = properties; + } + + /** + * Loads the catalog shipped on the classpath. + * + * @return The catalog. Never {@code null}. + * @throws IOException Thrown if the resource is missing or cannot be read. + */ + public static DictionaryCatalog loadDefault() throws IOException { + try (InputStream in = DictionaryCatalog.class.getClassLoader() + .getResourceAsStream(DEFAULT_RESOURCE)) { + if (in == null) { + throw new IOException("missing classpath resource " + DEFAULT_RESOURCE); + } + return load(in); + } + } + + /** + * Loads a catalog from a properties stream. + * + * @param in The properties content. Must not be {@code null}. + * @return The catalog. Never {@code null}. + * @throws IOException Thrown if reading fails. + * @throws IllegalArgumentException Thrown if {@code in} is {@code null}. + */ + public static DictionaryCatalog load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + final Properties properties = new Properties(); + properties.load(in); + return new DictionaryCatalog(properties); + } + + /** + * {@return the catalog entry ids, in encounter order} + */ + public Set ids() { + final Set ids = new LinkedHashSet<>(); + for (final String key : properties.stringPropertyNames()) { + if (key.endsWith(".url")) { + ids.add(key.substring(0, key.length() - ".url".length())); + } + } + return Collections.unmodifiableSet(ids); + } + + /** + * Looks up one catalog entry. + * + * @param id The entry id, for example {@code mecab.ipadic}. + * @return The entry. Never {@code null}. + * @throws IOException Thrown if the entry is incomplete or the URI is malformed. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null}. + */ + public Entry get(String id) throws IOException { + if (id == null) { + throw new IllegalArgumentException("id must not be null"); + } + final String url = properties.getProperty(id + ".url"); + final String sha512 = properties.getProperty(id + ".sha512"); + if (url == null || sha512 == null) { + throw new IOException("unknown or incomplete dictionary catalog entry: " + id); + } + final String filename = properties.getProperty(id + ".filename"); + try { + return new Entry(id, new URI(url), sha512.trim(), filename); + } catch (URISyntaxException e) { + throw new IOException("malformed catalog URI for " + id, e); + } + } + + /** + * Downloads a catalog entry into {@code target} after checking that remote catalog + * downloads are enabled. + * + * @param id The entry id. Must not be {@code null}. + * @param target The local file to create. Must not be {@code null}. + * @throws IOException Thrown if the property is not enabled, the entry is missing, + * or the download fails verification. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public void download(String id, Path target) throws IOException { + if (target == null) { + throw new IllegalArgumentException("target must not be null"); + } + if (!DownloadUtil.isRemoteDownloadEnabled()) { + throw new IOException("remote dictionary catalog downloads are disabled; set -D" + + DownloadUtil.REMOTE_DOWNLOAD_PROPERTY + "=true to enable"); + } + final Entry entry = get(id); + DownloadUtil.download(entry.uri(), target, entry.sha512()); + } + + /** + * One pinned remote file: a stable URL and the SHA-512 of its bytes. + * + * @param id The catalog id. + * @param uri The absolute download URI. + * @param sha512 The expected SHA-512 hex digest. + * @param filename An optional preferred local file name; may be {@code null}. + */ + public record Entry(String id, URI uri, String sha512, String filename) { + /** + * @param id The catalog id. Must not be {@code null}. + * @param uri The absolute download URI. Must not be {@code null}. + * @param sha512 The expected SHA-512 hex digest. Must not be {@code null}. + * @param filename An optional preferred local file name; may be {@code null}. + */ + public Entry { + if (id == null) { + throw new IllegalArgumentException("id must not be null"); + } + if (uri == null) { + throw new IllegalArgumentException("uri must not be null"); + } + if (sha512 == null) { + throw new IllegalArgumentException("sha512 must not be null"); + } + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java index 7554c064b3..44517e2389 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java @@ -21,11 +21,15 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLConnection; import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -38,6 +42,7 @@ import java.util.Formatter; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.regex.Matcher; @@ -51,7 +56,9 @@ import opennlp.tools.util.model.BaseModel; /** - * This class facilitates the downloading of pretrained OpenNLP models. + * Downloads remote resources into a local path: pretrained OpenNLP models, and any + * other file fetched through {@link #download(URI, Path, String)} with an expected + * SHA-512 digest. */ public class DownloadUtil { @@ -63,6 +70,22 @@ public class DownloadUtil { System.getProperty("OPENNLP_DOWNLOAD_MODEL_PATH", "models/ud-models-1.3/"); private static final String OPENNLP_DOWNLOAD_HOME = "OPENNLP_DOWNLOAD_HOME"; + /** + * System property that must be {@code true} before a + * {@link DictionaryCatalog} entry may be fetched. Explicit + * {@link #download(URI, Path, String)} calls do not require it: the caller already + * supplied the URI and digest. + */ + public static final String REMOTE_DOWNLOAD_PROPERTY = "opennlp.download.remote"; + + /** Inclusive ceiling on bytes buffered for one {@link #download(URI, Path, String)}. */ + public static final long MAX_DOWNLOAD_BYTES = 512L * 1024 * 1024; + + private static final int CONNECT_TIMEOUT_MS = 30_000; + private static final int READ_TIMEOUT_MS = 300_000; + private static final int SHA512_HEX_LENGTH = 128; + private static final String DOWNLOAD_SUFFIX = ".download"; + private static Map> availableModels; /** @@ -172,6 +195,151 @@ public static T downloadModel(URL url, Class type) thro } } + /** + * Downloads {@code source} into {@code target} and requires the SHA-512 digest of the + * stored bytes to equal {@code expectedSha512}. The download is written to a sibling + * temporary file and moved into place only after the digest matches. The transfer is + * capped at {@link #MAX_DOWNLOAD_BYTES}; remote {@code http} and {@code https} URIs + * additionally use connect and read timeouts. + * + * @param source The absolute URI to fetch. Must not be {@code null}. + * @param target The local file to create or replace. Must not be {@code null}. + * @param expectedSha512 The expected SHA-512 digest as 128 lowercase or uppercase hex + * digits. Must not be {@code null}. + * @throws IOException Thrown if fetching fails, the size ceiling is exceeded, or the + * digest does not match. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, {@code source} + * is not absolute, or {@code expectedSha512} is not 128 hex digits. + */ + public static void download(URI source, Path target, String expectedSha512) + throws IOException { + download(source, target, expectedSha512, MAX_DOWNLOAD_BYTES); + } + + /** + * Downloads {@code source} into {@code target} under a caller-supplied byte ceiling. + * + * @param source The absolute URI to fetch. Must not be {@code null}. + * @param target The local file to create or replace. Must not be {@code null}. + * @param expectedSha512 The expected SHA-512 digest as 128 hex digits. Must not be + * {@code null}. + * @param maxBytes The inclusive ceiling on bytes read from {@code source}. + * @throws IOException Thrown if fetching fails, {@code maxBytes} is exceeded, or the + * digest does not match. + * @throws IllegalArgumentException Thrown if a parameter is invalid, see + * {@link #download(URI, Path, String)}. + */ + static void download(URI source, Path target, String expectedSha512, long maxBytes) + throws IOException { + if (source == null) { + throw new IllegalArgumentException("source must not be null"); + } + if (target == null) { + throw new IllegalArgumentException("target must not be null"); + } + if (expectedSha512 == null) { + throw new IllegalArgumentException("expectedSha512 must not be null"); + } + if (!source.isAbsolute()) { + throw new IllegalArgumentException("source must be an absolute URI"); + } + final String normalized = normalizeSha512(expectedSha512); + final Path parent = target.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + final Path partial = target.resolveSibling(target.getFileName() + DOWNLOAD_SUFFIX); + Files.deleteIfExists(partial); + try { + long size = 0L; + final MessageDigest digest = sha512Digest(); + final URLConnection connection = open(source); + try (InputStream in = connection.getInputStream(); + DigestInputStream digester = new DigestInputStream(in, digest); + OutputStream out = Files.newOutputStream(partial)) { + final byte[] buffer = new byte[8192]; + int n; + while ((n = digester.read(buffer)) >= 0) { + size += n; + if (size > maxBytes) { + throw new IOException("download size exceeds safe limit of " + maxBytes); + } + out.write(buffer, 0, n); + } + } finally { + if (connection instanceof HttpURLConnection http) { + http.disconnect(); + } + } + final String actual = byteArrayToHexString(digest.digest()); + if (!actual.equals(normalized)) { + throw new IOException("SHA512 checksum validation failed for " + target.getFileName() + + ". Expected: " + normalized + ", but got: " + actual); + } + try { + Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + Files.deleteIfExists(partial); + throw e; + } + } + + /** + * {@return {@code true} when {@link #REMOTE_DOWNLOAD_PROPERTY} is the string + * {@code true}, ignoring case} + */ + public static boolean isRemoteDownloadEnabled() { + return Boolean.parseBoolean(System.getProperty(REMOTE_DOWNLOAD_PROPERTY)); + } + + /** + * Opens a connection to {@code source} with connect and read timeouts applied. + * + * @param source The absolute URI to connect to. + * @return The configured, not yet connected, connection. + * @throws IOException Thrown if no connection can be created for {@code source}. + */ + private static URLConnection open(URI source) throws IOException { + final URLConnection connection = source.toURL().openConnection(); + connection.setConnectTimeout(CONNECT_TIMEOUT_MS); + connection.setReadTimeout(READ_TIMEOUT_MS); + return connection; + } + + /** + * Trims and lowercases a SHA-512 hex digest. + * + * @param expectedSha512 The digest to normalize. + * @return The digest as 128 lowercase hex digits. + * @throws IllegalArgumentException Thrown if the digest is not 128 hex digits. + */ + private static String normalizeSha512(String expectedSha512) { + final String hex = expectedSha512.trim().toLowerCase(Locale.ROOT); + if (hex.length() != SHA512_HEX_LENGTH || !hex.chars().allMatch( + c -> c >= '0' && c <= '9' || c >= 'a' && c <= 'f')) { + throw new IllegalArgumentException( + "expectedSha512 must be 128 hexadecimal digits"); + } + return hex; + } + + /** + * {@return a fresh SHA-512 {@link MessageDigest}} + * + * @throws IOException Thrown if the JVM does not provide the algorithm. + */ + private static MessageDigest sha512Digest() throws IOException { + try { + return MessageDigest.getInstance("SHA-512"); + } catch (NoSuchAlgorithmException e) { + throw new IOException("SHA-512 algorithm not found", e); + } + } + public static Map> getAvailableModels() { if (availableModels == null) { try { diff --git a/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties new file mode 100644 index 0000000000..b016826d89 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pinned remote dictionary files. OpenNLP ships URLs and SHA-512 digests only; +# the data itself is never bundled. Fetching requires -Dopennlp.download.remote=true. + +# MeCab IPADIC 2.7.0 (EUC-JP). Upstream: MeCab project on SourceForge. +mecab.ipadic.url=https://downloads.sourceforge.net/project/mecab/\ +mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz +mecab.ipadic.sha512=35ea662cb62f1967849f7ed5781bd6dafef0fe20d63e88d9\ +a0057666e57ed23d5a0e6fb8d0701a0cc4da43a1050c1b0246\ +3bb862decc71c36b7fc2acdc158d86 +mecab.ipadic.filename=mecab-ipadic-2.7.0-20070801.tar.gz + +# mecab-ko-dic 2.1.1 (UTF-8). Upstream: eunjeon/mecab-ko-dic on Bitbucket. +mecab.ko-dic.url=https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/\ +mecab-ko-dic-2.1.1-20180720.tar.gz +mecab.ko-dic.sha512=986f8f9c66c53accd296756bf632c979d2d44b695ada33f3\ +6c662f210dba34cd95d67b61dd8c84a1f7d59f80ee6bc22eb1\ +e9afb5dc6a7f9b6b75b4fbf2f8164f +mecab.ko-dic.filename=mecab-ko-dic-2.1.1-20180720.tar.gz + +# LibreOffice en_US Hunspell pair, pinned to dictionaries commit 208a9fd8. +hunspell.en_US.aff.url=https://raw.githubusercontent.com/LibreOffice/\ +dictionaries/208a9fd80b2a182fe20f224cd615119c6323ae2e/en/en_US.aff +hunspell.en_US.aff.sha512=2b4448dfdff03caf300914415f4642f8d2ba5b650c5f024a\ +12355b420a279ffc12146649fce092ba591504476634a3d6\ +fd4c079335a27085b396fa76bfd28b74 +hunspell.en_US.aff.filename=en_US.aff + +hunspell.en_US.dic.url=https://raw.githubusercontent.com/LibreOffice/\ +dictionaries/208a9fd80b2a182fe20f224cd615119c6323ae2e/en/en_US.dic +hunspell.en_US.dic.sha512=4be737249a8a436d20a02be575dcf6cf2f06f5f2abb840ea\ +5ec0ef0ac73a71fa0e4669e527c703d5c6b50ef61713a674\ +1b55bc136d559a54bcdeebcd62027988 +hunspell.en_US.dic.filename=en_US.dic + +hunspell.en_US.readme.url=https://raw.githubusercontent.com/LibreOffice/\ +dictionaries/208a9fd80b2a182fe20f224cd615119c6323ae2e/en/README_en_US.txt +hunspell.en_US.readme.sha512=aa23ebc8adc0649b540264c7bf98cef5b6e383fec0e4a1a7\ +dd49d1c887cfeefd8edf6a568afc8a651521a3864c5b1ab5\ +0748ad16230d386809080b5b09135082 +hunspell.en_US.readme.filename=README_en_US.txt diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index 0fc58585de..7fadcc5cbf 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.DigestTestUtil; import opennlp.tools.util.Span; /** @@ -87,10 +88,11 @@ void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) final Path archiveFile = work.resolve("mini-dict-0.1.tar.gz"); Files.write(archiveFile, archive); - // Install: fetch the archive from the user-chosen location and unpack the payload. + // Install: fetch the archive from the user-chosen location, verify its digest, + // and unpack the payload. final Path dictionaryDirectory = work.resolve("dictionary"); - final int extracted = - MecabDictionaryInstaller.install(archiveFile.toUri(), dictionaryDirectory); + final int extracted = MecabDictionaryInstaller.install( + archiveFile.toUri(), dictionaryDirectory, DigestTestUtil.sha512(archive)); Assertions.assertEquals(4, extracted); // Load and tokenize; both views must agree and stay in original coordinates. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index 5141025d8b..b2931a9621 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -27,6 +28,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.DigestTestUtil; +import opennlp.tools.util.DownloadUtil; + /** * Tests the installer against project-authored, in-memory archives; no external * dictionary data and no network access are involved. @@ -73,6 +77,49 @@ void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) Assertions.assertTrue(Files.exists(target.resolve("words.csv"))); } + @Test + void testRemoteInstallWithoutDigestFailsLoud(@TempDir Path target) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> MecabDictionaryInstaller.install( + URI.create("https://example.invalid/dict.tar.gz"), target)); + } + + @Test + void testInstallVerifiesDigest(@TempDir Path source, @TempDir Path target) + throws Exception { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { + {"d/words.csv", "cat,0,0,100,noun\n"}, + {"d/matrix.def", "1 1\n0 0 0\n"}}); + final Path archiveFile = source.resolve("dict.tar.gz"); + Files.write(archiveFile, archive); + + final int extracted = MecabDictionaryInstaller.install( + archiveFile.toUri(), target, DigestTestUtil.sha512(archive)); + Assertions.assertEquals(2, extracted); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionaryInstaller.install(archiveFile.toUri(), + target.resolve("other"), DigestTestUtil.sha512(new byte[] {1}))); + Assertions.assertTrue(e.getMessage().contains("SHA512 checksum validation failed")); + } + + @Test + void testInstallFromCatalogRequiresRemoteProperty(@TempDir Path target) { + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + try { + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionaryInstaller.installFromCatalog("mecab.ipadic", target)); + Assertions.assertTrue(e.getMessage().contains(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY)); + } finally { + if (previous == null) { + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + } else { + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, previous); + } + } + } + @Test void testArchivesWithoutDictionaryFilesFailLoud(@TempDir Path target) throws IOException { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java new file mode 100644 index 0000000000..fd05fd6ed5 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests the opt-in dictionary catalog against an in-memory properties file and a + * local file URI so no network access is required. + */ +public class DictionaryCatalogTest { + + @Test + void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception { + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + final DictionaryCatalog loaded = demoCatalog(dir, payload); + + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + try { + final IOException e = Assertions.assertThrows(IOException.class, + () -> loaded.download("demo", dir.resolve("out.bin"))); + Assertions.assertTrue(e.getMessage().contains(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY)); + } finally { + restore(previous); + } + } + + @Test + void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception { + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + final DictionaryCatalog loaded = demoCatalog(dir, payload); + final Path target = dir.resolve("out.bin"); + + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, "true"); + try { + loaded.download("demo", target); + Assertions.assertArrayEquals(payload, Files.readAllBytes(target)); + } finally { + restore(previous); + } + } + + @Test + void testDefaultCatalogContainsMecabAndHunspellEntries() throws IOException { + final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); + Assertions.assertTrue(catalog.ids().contains("mecab.ipadic")); + Assertions.assertTrue(catalog.ids().contains("mecab.ko-dic")); + Assertions.assertTrue(catalog.ids().contains("hunspell.en_US.aff")); + Assertions.assertEquals(128, catalog.get("mecab.ipadic").sha512().length()); + Assertions.assertEquals(128, catalog.get("hunspell.en_US.dic").sha512().length()); + } + + private static DictionaryCatalog demoCatalog(Path dir, byte[] payload) + throws IOException { + final Path source = dir.resolve("dict.bin"); + Files.write(source, payload); + final String catalog = "demo.url=" + source.toUri() + "\n" + + "demo.sha512=" + DigestTestUtil.sha512(payload) + "\n"; + return DictionaryCatalog.load( + new ByteArrayInputStream(catalog.getBytes(StandardCharsets.UTF_8))); + } + + private static void restore(String previous) { + if (previous == null) { + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + } else { + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, previous); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java new file mode 100644 index 0000000000..af85b09a8e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** + * Computes SHA-512 digests for test fixtures. + */ +public final class DigestTestUtil { + + private DigestTestUtil() { + } + + /** + * {@return the SHA-512 digest of {@code bytes} as 128 lowercase hex digits} + * + * @param bytes The content to digest. Must not be {@code null}. + */ + public static String sha512(byte[] bytes) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-512").digest(bytes)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java new file mode 100644 index 0000000000..32510fcb35 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Pins {@link DownloadUtil#download(java.net.URI, Path, String)} against local file URIs + * so digest verification and the size ceiling are covered without a network. + */ +public class DownloadUtilFileTest { + + private static final byte[] PAYLOAD = "dictionary-bytes".getBytes(StandardCharsets.UTF_8); + + @Test + void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + DownloadUtil.download(source.toUri(), target, DigestTestUtil.sha512(PAYLOAD)); + + Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); + } + + @Test + void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + final String wrong = DigestTestUtil.sha512("other".getBytes(StandardCharsets.UTF_8)); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> DownloadUtil.download(source.toUri(), target, wrong)); + Assertions.assertTrue(e.getMessage().contains("SHA512 checksum validation failed")); + Assertions.assertTrue(Files.notExists(target)); + } + + @Test + void testDownloadRequiresSha512() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> DownloadUtil.download(Path.of("x").toUri(), Path.of("y"), null)); + } + + @Test + void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> DownloadUtil.download(source.toUri(), dir.resolve("target.bin"), "abc123")); + } + + @Test + void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> DownloadUtil.download(source.toUri(), target, + DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length - 1)); + Assertions.assertTrue(e.getMessage().contains("exceeds safe limit")); + Assertions.assertTrue(Files.notExists(target)); + } + + @Test + void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + DownloadUtil.download(source.toUri(), target, + DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length); + + Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); + } +} diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index 1b376abc67..c59973553f 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -546,7 +546,10 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { emits the cheapest segmentation with spans in original text coordinates. UnigramSegmenter does the same from a plain frequency lexicon. Install a dictionary archive with MecabDictionaryInstaller, load it - as a MecabDictionary, and tokenize. + as a MecabDictionary, and tokenize. Remote archives require an + expected SHA-512 digest; the opt-in catalog + (installFromCatalog) also needs + -Dopennlp.download.remote=true. matrix.def must list a cost for every declared cell, and matrix dimensions plus lexicon size are bounded by the shared ResourceLimits.MAX_ENTRIES limit. @@ -556,7 +559,7 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { LatticeUsageExampleTest asserts the install-load-tokenize and lexicon flows shown here. - Date: Thu, 6 Aug 2026 16:12:05 -0400 Subject: [PATCH 18/24] OPENNLP-1894: Make download and extraction byte budgets overridable at startup Keep the 512 MiB download and tar-entry ceilings and the 2 GiB total extraction ceiling as defaults, but read them from the system properties opennlp.download.max.bytes, opennlp.install.max.entry.bytes, and opennlp.install.max.total.bytes at class load, so dictionaries larger than the defaults, such as UniDic, install without a code change. Absent or invalid values fall back to the defaults. Tests pin the parsing and the defaults; the README and manual document the overrides. --- dev/README-mecab-dictionaries.md | 16 ++++++++ .../lattice/MecabDictionaryInstaller.java | 38 +++++++++++++++--- .../java/opennlp/tools/util/DownloadUtil.java | 40 ++++++++++++++++++- .../lattice/MecabDictionaryInstallerTest.java | 7 ++++ .../tools/util/DownloadUtilFileTest.java | 36 +++++++++++++++++ opennlp-docs/src/docbkx/tokenizer.xml | 8 +++- 6 files changed, 137 insertions(+), 8 deletions(-) diff --git a/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md index 5284e43560..c30cec81ec 100644 --- a/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -69,6 +69,22 @@ A local `file:` URI may omit the digest: `MecabDictionaryInstaller.install(localArchive.toUri(), targetDirectory)`. Any other URI scheme requires the digest. +## Size budgets for larger dictionaries + +Downloads and extraction are bounded so a crafted archive cannot fill the disk: by +default one download is capped at 512 MiB, one extracted tar entry at 512 MiB, and +the total extracted payload at 2 GiB. IPADIC and mecab-ko-dic fit comfortably. For +larger dictionaries, such as UniDic, raise the ceilings at JVM startup: + +```bash +-Dopennlp.download.max.bytes=4294967296 \ +-Dopennlp.install.max.entry.bytes=4294967296 \ +-Dopennlp.install.max.total.bytes=8589934592 +``` + +Values must be positive byte counts; anything absent or invalid falls back to the +default. + ## Load and tokenize `MecabDictionary.load(Path)` assumes UTF-8. IPADIC needs the two-argument overload: diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index b662ca00b2..0566a2158b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -49,7 +49,10 @@ * *

Extraction is bounded: each entry's declared size, the total bytes written, the * number of extracted dictionary files, and the gzip expansion ratio each have an - * explicit ceiling so a crafted archive cannot fill the disk.

+ * explicit ceiling so a crafted archive cannot fill the disk. The byte ceilings can be + * raised at JVM startup via {@link #MAX_ENTRY_BYTES_PROPERTY} and + * {@link #MAX_TOTAL_EXTRACTED_BYTES_PROPERTY} for dictionaries larger than the + * defaults, such as UniDic.

* * @since 3.0.0 */ @@ -61,11 +64,36 @@ public final class MecabDictionaryInstaller { private static final int TAR_SIZE_LENGTH = 12; private static final int TAR_TYPE_OFFSET = 156; - /** Inclusive ceiling on one tar entry's declared size, in bytes. */ - static final long MAX_ENTRY_BYTES = 512L * 1024 * 1024; + /** + * System property for overriding {@link #MAX_ENTRY_BYTES}. Set at JVM startup, + * e.g. {@code -Dopennlp.install.max.entry.bytes=2147483648} for dictionaries whose + * lexicon files exceed the default ceiling. Falls back to the default if absent, + * non-numeric, or not positive. + */ + public static final String MAX_ENTRY_BYTES_PROPERTY = "opennlp.install.max.entry.bytes"; + + /** + * System property for overriding {@link #MAX_TOTAL_EXTRACTED_BYTES}. Set at JVM + * startup, e.g. {@code -Dopennlp.install.max.total.bytes=8589934592}. Falls back to + * the default if absent, non-numeric, or not positive. + */ + public static final String MAX_TOTAL_EXTRACTED_BYTES_PROPERTY = + "opennlp.install.max.total.bytes"; - /** Inclusive ceiling on the sum of extracted dictionary file sizes, in bytes. */ - static final long MAX_TOTAL_EXTRACTED_BYTES = 2L * 1024 * 1024 * 1024; + /** + * Inclusive ceiling on one tar entry's declared size, in bytes: 512 MiB unless + * overridden via {@link #MAX_ENTRY_BYTES_PROPERTY}. + */ + static final long MAX_ENTRY_BYTES = + DownloadUtil.configuredLimit(MAX_ENTRY_BYTES_PROPERTY, 512L * 1024 * 1024); + + /** + * Inclusive ceiling on the sum of extracted dictionary file sizes, in bytes: 2 GiB + * unless overridden via {@link #MAX_TOTAL_EXTRACTED_BYTES_PROPERTY}. + */ + static final long MAX_TOTAL_EXTRACTED_BYTES = + DownloadUtil.configuredLimit(MAX_TOTAL_EXTRACTED_BYTES_PROPERTY, + 2L * 1024 * 1024 * 1024); /** Inclusive ceiling on the number of dictionary files extracted from one archive. */ static final int MAX_EXTRACTED_ENTRIES = 10_000; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java index 44517e2389..41593e97d4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java @@ -78,8 +78,20 @@ public class DownloadUtil { */ public static final String REMOTE_DOWNLOAD_PROPERTY = "opennlp.download.remote"; - /** Inclusive ceiling on bytes buffered for one {@link #download(URI, Path, String)}. */ - public static final long MAX_DOWNLOAD_BYTES = 512L * 1024 * 1024; + /** + * System property for overriding {@link #MAX_DOWNLOAD_BYTES}. Set at JVM startup, + * e.g. {@code -Dopennlp.download.max.bytes=2147483648} for dictionaries larger than + * the default ceiling. Falls back to the default if absent, non-numeric, or not + * positive. + */ + public static final String MAX_DOWNLOAD_BYTES_PROPERTY = "opennlp.download.max.bytes"; + + /** + * Inclusive ceiling on bytes buffered for one {@link #download(URI, Path, String)}, + * 512 MiB unless overridden via {@link #MAX_DOWNLOAD_BYTES_PROPERTY}. + */ + public static final long MAX_DOWNLOAD_BYTES = + configuredLimit(MAX_DOWNLOAD_BYTES_PROPERTY, 512L * 1024 * 1024); private static final int CONNECT_TIMEOUT_MS = 30_000; private static final int READ_TIMEOUT_MS = 300_000; @@ -296,6 +308,30 @@ public static boolean isRemoteDownloadEnabled() { return Boolean.parseBoolean(System.getProperty(REMOTE_DOWNLOAD_PROPERTY)); } + /** + * Reads a byte-budget override from a system property. Budget constants are + * initialized from it once at class load, so overrides must be set at JVM startup. + * + * @param property The system property name to read. + * @param fallback The value to use when the property is absent or invalid. + * @return The property's value when it parses as a positive {@code long}, otherwise + * {@code fallback}. + */ + public static long configuredLimit(String property, long fallback) { + final String value = System.getProperty(property, "").trim(); + if (!value.isEmpty()) { + try { + final long parsed = Long.parseLong(value); + if (parsed > 0) { + return parsed; + } + } catch (NumberFormatException ignore) { + // Fall through to the default. + } + } + return fallback; + } + /** * Opens a connection to {@code source} with connect and read timeouts applied. * diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index b2931a9621..70fdf8048b 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -211,4 +211,11 @@ void testInvalidArguments(@TempDir Path target) { () -> MecabDictionaryInstaller.extract( new ByteArrayInputStream(new byte[0]), null)); } + + @Test + void testDefaultBudgetsWithoutOverrides() { + Assertions.assertEquals(512L * 1024 * 1024, MecabDictionaryInstaller.MAX_ENTRY_BYTES); + Assertions.assertEquals(2L * 1024 * 1024 * 1024, + MecabDictionaryInstaller.MAX_TOTAL_EXTRACTED_BYTES); + } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java index 32510fcb35..113f1689cf 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -97,4 +97,40 @@ void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); } + + @Test + void testConfiguredLimitOverridesFromProperty() { + final String property = "opennlp.test.limit.override"; + System.setProperty(property, "1024"); + try { + Assertions.assertEquals(1024L, DownloadUtil.configuredLimit(property, 7L)); + } finally { + System.clearProperty(property); + } + } + + @Test + void testConfiguredLimitFallsBackWhenAbsent() { + Assertions.assertEquals(7L, + DownloadUtil.configuredLimit("opennlp.test.limit.absent", 7L)); + } + + @Test + void testConfiguredLimitRejectsInvalidValues() { + final String property = "opennlp.test.limit.invalid"; + for (final String invalid : new String[] {"", " ", "abc", "-1", "0"}) { + System.setProperty(property, invalid); + try { + Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L), + "value <" + invalid + "> must fall back"); + } finally { + System.clearProperty(property); + } + } + } + + @Test + void testDefaultBudgetsWithoutOverrides() { + Assertions.assertEquals(512L * 1024 * 1024, DownloadUtil.MAX_DOWNLOAD_BYTES); + } } diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index c59973553f..a81dea5d3f 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -553,7 +553,13 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { matrix.def must list a cost for every declared cell, and matrix dimensions plus lexicon size are bounded by the shared ResourceLimits.MAX_ENTRIES limit. - Extraction caps per-entry size, total bytes, entry count, and gzip expansion. + Extraction caps per-entry size, total bytes, entry count, and gzip expansion; + the byte ceilings default to 512 MiB per download and entry and 2 GiB total, + and can be raised at JVM startup via + opennlp.download.max.bytes, + opennlp.install.max.entry.bytes, and + opennlp.install.max.total.bytes for larger dictionaries such as + UniDic. Load also rejects an unk.def template for a category char.def never defined, and accepts MeCab-quoted CSV fields. LatticeUsageExampleTest asserts the install-load-tokenize and From ec43621bb68bfb808eb3964c92db44227f554b10 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 19:11:44 -0400 Subject: [PATCH 19/24] OPENNLP-1894: Trigger CI for the budget-override commit From f171dcad11c6281ba4388a2367e6979db466899e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:58:20 -0400 Subject: [PATCH 20/24] OPENNLP-1894: Address review: cite named formats and align test conventions Link MeCab, IPADIC, UniDic, mecab-ko-dic, and the POSIX ustar format where the javadoc names them, and write MeCab in its own casing across the prose. Document the entryCount parameter and its limit rejection on MecabDictionary.readLexicon. Replace three literal kanji in LatticeTokenizerTest with Unicode escapes, matching the file's ASCII-only convention. Convert DownloadUtilFileTest's invalid-limit loop to a parameterized test so a failing value is identifiable. --- dev/README-mecab-dictionaries.md | 6 +++--- .../tokenize/lattice/MecabDictionary.java | 19 ++++++++++++------ .../lattice/MecabDictionaryInstaller.java | 10 ++++++---- .../lattice/LatticeTokenizerTest.java | 6 +++--- .../lattice/LatticeUsageExampleTest.java | 6 +++--- .../tools/util/DownloadUtilFileTest.java | 20 +++++++++---------- 6 files changed, 38 insertions(+), 29 deletions(-) diff --git a/dev/README-mecab-dictionaries.md b/dev/README-mecab-dictionaries.md index c30cec81ec..310183ea68 100644 --- a/dev/README-mecab-dictionaries.md +++ b/dev/README-mecab-dictionaries.md @@ -17,9 +17,9 @@ # CJK dictionaries for the lattice tokenizer -The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a mecab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and each dictionary carries its own license. Read the license file inside the archive before use. +The lattice tokenizer (`opennlp.tools.tokenize.lattice`) segments Japanese and Korean over a MeCab-format dictionary, and the unigram segmenter handles Chinese over a plain word-frequency lexicon. Apache OpenNLP bundles no dictionary data: you download a dictionary from the project of your choice, and each dictionary carries its own license. Read the license file inside the archive before use. -## Known mecab-format dictionary projects +## Known MeCab-format dictionary projects | Catalog id | Dictionary | Language | Encoding | |---|---|---|---| @@ -108,7 +108,7 @@ threads, so load once and reuse. ## Chinese: the unigram segmenter needs only a frequency lexicon -`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use mecab dictionaries. It +`opennlp.tools.tokenize.lattice.UnigramSegmenter` does not use MeCab dictionaries. It loads a plain text lexicon, one entry per line: the word, its count, and optionally a tag, separated by whitespace. Any word-frequency list you have the rights to use works: diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index dddbfc1296..8ab1787505 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -36,15 +36,19 @@ import opennlp.tools.util.StringUtil; /** - * An immutable, in-memory dictionary in the mecab directory format: lexicon entries + * An immutable, in-memory dictionary in the + * MeCab directory format: lexicon entries * from the {@code *.csv} files, connection costs from {@code matrix.def}, character * categories from {@code char.def}, and unknown-word templates from {@code unk.def}, * loaded from a user-supplied dictionary directory. No dictionary data is bundled or * downloaded by this class. * - *

The same format serves multiple languages: the Japanese IPADIC and UniDic - * distributions and the Korean mecab-ko-dic all load through this one reader, with the - * feature columns passed through untouched because their schemas differ.

+ *

The same format serves multiple languages: the Japanese + * IPADIC and + * UniDic distributions and the Korean + * mecab-ko-dic all load + * through this one reader, with the feature columns passed through untouched because + * their schemas differ.

* *

Each instance keeps about 0.75 MB of category tables keyed by the 16-bit code-unit * space, so load once and share. Lexicon CSV files under the dictionary directory are @@ -705,8 +709,11 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc * ids. * @param rightSize The second {@code matrix.def} dimension, which bounds left context * ids. - * @throws IOException Thrown if the file is missing, an entry is malformed, or an - * entry's context id is outside the matrix dimensions. + * @param entryCount A one-element running total of entries read so far, shared across + * the lexicon files of one load. + * @throws IOException Thrown if the file is missing, an entry is malformed, an + * entry's context id is outside the matrix dimensions, or the running entry + * count exceeds {@link ResourceLimits#MAX_ENTRIES}. */ private static void readLexicon(Path file, Charset charset, Map> target, int leftSize, int rightSize, int[] entryCount) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 0566a2158b..16e12e3530 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -32,15 +32,17 @@ import opennlp.tools.util.model.UncloseableInputStream; /** - * Fetches and unpacks a mecab-format dictionary archive into a local directory, so the + * Fetches and unpacks a MeCab-format dictionary archive into a local directory, so the * dictionary is acquired by the user at install time and never ships with this library. * No dictionary data is bundled. Any non-{@code file:} archive is downloaded through * {@link DownloadUtil#download(URI, Path, String)} and requires an expected SHA-512 * digest. Built-in catalog URLs are opt-in via {@link #installFromCatalog(String, Path)}. * - *

The installer reads gzip-compressed ustar archives (POSIX.1-1988), the format the - * common distributions use. GNU long-name ({@code L}) and PAX ({@code x}/{@code g}) - * headers are not supported; entry names must fit the 100-byte ustar name field. It + *

The installer reads gzip-compressed + * + * ustar archives (POSIX.1-1988), the format the common distributions use. GNU + * long-name ({@code L}) and PAX ({@code x}/{@code g}) headers are not supported; entry + * names must fit the 100-byte ustar name field. It * extracts only the dictionary payload: the {@code *.csv} lexicon files and * {@code *.def} definition files that a {@link MecabDictionary} reads, plus the * {@code dicrc} configuration file distributions ship alongside them. Entries are diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index 324bf9e058..f2acef5238 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -619,16 +619,16 @@ void testInvalidArguments() { void testMorphemeRejectsInvalidArguments() { final Span span = new Span(0, 1); Assertions.assertThrows(IllegalArgumentException.class, - () -> new Morpheme(null, "東", List.of("noun"), false)); + () -> new Morpheme(null, "\u6771", List.of("noun"), false)); Assertions.assertThrows(IllegalArgumentException.class, () -> new Morpheme(span, null, List.of("noun"), false)); Assertions.assertThrows(IllegalArgumentException.class, () -> new Morpheme(span, "", List.of("noun"), false)); Assertions.assertThrows(IllegalArgumentException.class, - () -> new Morpheme(span, "東", null, false)); + () -> new Morpheme(span, "\u6771", null, false)); final List features = new ArrayList<>(List.of("noun")); - final Morpheme morpheme = new Morpheme(span, "東", features, false); + final Morpheme morpheme = new Morpheme(span, "\u6771", features, false); features.add("proper"); Assertions.assertEquals(List.of("noun"), morpheme.features()); } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java index 7fadcc5cbf..897d965dd5 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeUsageExampleTest.java @@ -33,7 +33,7 @@ /** * Demonstrates the intended end-to-end usage of this package with miniature, - * project-authored data: a mecab-format dictionary archive is installed with + * project-authored data: a MeCab-format dictionary archive is installed with * {@link MecabDictionaryInstaller}, loaded as a {@link MecabDictionary}, and segmented * with a {@link LatticeTokenizer}; a plain frequency lexicon is loaded and segmented * with a {@link UnigramSegmenter}. Everything is written to a temporary directory by @@ -50,7 +50,7 @@ public class LatticeUsageExampleTest { /** - * Walks the full mecab-format flow: package a miniature Japanese dictionary as a + * Walks the full MeCab-format flow: package a miniature Japanese dictionary as a * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The * segmentation must pick the cheaper path (Tokyo plus the metropolis suffix) over * the competing reading (east plus Kyoto), the spans must be in original text @@ -60,7 +60,7 @@ public class LatticeUsageExampleTest { void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) throws IOException { // A minimal but complete dictionary: one lexicon file plus the three definition - // files every mecab-format distribution contains, wrapped like a release archive. + // files every MeCab-format distribution contains, wrapped like a release archive. final byte[] archive = TarGzArchives.gzippedTar(new String[][] { {"mini-dict-0.1/lexicon.csv", String.join("\n", "\u6771\u4EAC,0,0,3000,noun,proper", diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java index 113f1689cf..f00171c2fa 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Pins {@link DownloadUtil#download(java.net.URI, Path, String)} against local file URIs @@ -115,17 +117,15 @@ void testConfiguredLimitFallsBackWhenAbsent() { DownloadUtil.configuredLimit("opennlp.test.limit.absent", 7L)); } - @Test - void testConfiguredLimitRejectsInvalidValues() { + @ParameterizedTest(name = "value \"{0}\" falls back") + @ValueSource(strings = {"", " ", "abc", "-1", "0"}) + void testConfiguredLimitRejectsInvalidValues(String invalid) { final String property = "opennlp.test.limit.invalid"; - for (final String invalid : new String[] {"", " ", "abc", "-1", "0"}) { - System.setProperty(property, invalid); - try { - Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L), - "value <" + invalid + "> must fall back"); - } finally { - System.clearProperty(property); - } + System.setProperty(property, invalid); + try { + Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L)); + } finally { + System.clearProperty(property); } } From 163e3960d1776d49e329846b42b56685804b9b5d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:19:44 -0400 Subject: [PATCH 21/24] OPENNLP-1894: Reconcile the shared download test files with the sibling PR Both PRs carry byte-identical copies; this folds the sibling's test javadoc into this side's parameterized fixtures so the copies match again. --- .../tools/util/DictionaryCatalogTest.java | 34 +++++++++++++++ .../tools/util/DownloadUtilFileTest.java | 41 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java index fd05fd6ed5..d721873951 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java @@ -33,6 +33,13 @@ */ public class DictionaryCatalogTest { + /** + * Verifies that a catalog download without the remote-download property fails with + * the property name in the message. + * + * @param dir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture catalog cannot be prepared. + */ @Test void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception { final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); @@ -49,6 +56,13 @@ void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception { } } + /** + * Verifies that an enabled catalog download fetches the entry and writes the + * digest-verified bytes to the target. + * + * @param dir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture catalog cannot be prepared or fetched. + */ @Test void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception { final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); @@ -65,6 +79,12 @@ void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception { } } + /** + * Verifies that the shipped catalog holds the MeCab and Hunspell entries, each with + * a full-length SHA-512 digest. + * + * @throws IOException Thrown if the shipped catalog fails to load. + */ @Test void testDefaultCatalogContainsMecabAndHunspellEntries() throws IOException { final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); @@ -75,6 +95,15 @@ void testDefaultCatalogContainsMecabAndHunspellEntries() throws IOException { Assertions.assertEquals(128, catalog.get("hunspell.en_US.dic").sha512().length()); } + /** + * Builds a one-entry catalog whose URL is a local file holding {@code payload}, so + * downloads need no network. + * + * @param dir The directory to write the payload file into. + * @param payload The bytes the catalog entry points at. + * @return The loaded catalog. Never {@code null}. + * @throws IOException Thrown if the payload file cannot be written. + */ private static DictionaryCatalog demoCatalog(Path dir, byte[] payload) throws IOException { final Path source = dir.resolve("dict.bin"); @@ -85,6 +114,11 @@ private static DictionaryCatalog demoCatalog(Path dir, byte[] payload) new ByteArrayInputStream(catalog.getBytes(StandardCharsets.UTF_8))); } + /** + * Restores the remote-download property to its value before the test. + * + * @param previous The saved value, or {@code null} when the property was unset. + */ private static void restore(String previous) { if (previous == null) { System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java index f00171c2fa..5451cabc5f 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -34,8 +34,16 @@ */ public class DownloadUtilFileTest { + /** The fixture bytes the download tests serve and digest. */ private static final byte[] PAYLOAD = "dictionary-bytes".getBytes(StandardCharsets.UTF_8); + /** + * Verifies that a download whose bytes match the expected digest lands in the + * target file. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or fetched. + */ @Test void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -47,6 +55,13 @@ void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException { Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); } + /** + * Verifies that a digest mismatch fails the download and leaves no target file + * behind. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ @Test void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -60,12 +75,20 @@ void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException { Assertions.assertTrue(Files.notExists(target)); } + /** Verifies that a {@code null} digest is rejected with the documented exception. */ @Test void testDownloadRequiresSha512() { Assertions.assertThrows(IllegalArgumentException.class, () -> DownloadUtil.download(Path.of("x").toUri(), Path.of("y"), null)); } + /** + * Verifies that a digest shorter than 128 hex digits is rejected before anything is + * fetched. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ @Test void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -75,6 +98,13 @@ void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException { () -> DownloadUtil.download(source.toUri(), dir.resolve("target.bin"), "abc123")); } + /** + * Verifies that a source larger than the byte ceiling fails the download and leaves + * no target file behind. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ @Test void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -88,6 +118,13 @@ void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException { Assertions.assertTrue(Files.notExists(target)); } + /** + * Pins the inclusive byte ceiling: a source of exactly the ceiling's size still + * downloads. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or fetched. + */ @Test void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -100,6 +137,7 @@ void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); } + /** Verifies that a positive property value overrides the fallback limit. */ @Test void testConfiguredLimitOverridesFromProperty() { final String property = "opennlp.test.limit.override"; @@ -111,12 +149,14 @@ void testConfiguredLimitOverridesFromProperty() { } } + /** Verifies that an unset property falls back to the given default. */ @Test void testConfiguredLimitFallsBackWhenAbsent() { Assertions.assertEquals(7L, DownloadUtil.configuredLimit("opennlp.test.limit.absent", 7L)); } + /** Verifies that blank, non-numeric, and non-positive values fall back. */ @ParameterizedTest(name = "value \"{0}\" falls back") @ValueSource(strings = {"", " ", "abc", "-1", "0"}) void testConfiguredLimitRejectsInvalidValues(String invalid) { @@ -129,6 +169,7 @@ void testConfiguredLimitRejectsInvalidValues(String invalid) { } } + /** Pins the default download ceiling of 512 MiB when no override property is set. */ @Test void testDefaultBudgetsWithoutOverrides() { Assertions.assertEquals(512L * 1024 * 1024, DownloadUtil.MAX_DOWNLOAD_BYTES); From 8e14a692c4fb1eab723bf358fa701ef3aa1cd6b0 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 16:57:53 -0400 Subject: [PATCH 22/24] OPENNLP-1894: Add a remote-gated end-to-end test over the catalog dictionaries Downloads the two pinned distributions with digest verification, installs them, loads them, and checks segmentation. Opt-in via -Dopennlp.download.remote=true like the catalog itself; CI never touches the network. At the previous tip the mecab-ko-dic case failed twice: the matrix cell bound rejected its genuine 3822 x 2693 matrix, and the flattened user-dic templates broke the lexicon parse. --- .../lattice/MecabCatalogEndToEndTest.java | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabCatalogEndToEndTest.java diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabCatalogEndToEndTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabCatalogEndToEndTest.java new file mode 100644 index 0000000000..5cb6e678b9 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabCatalogEndToEndTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Downloads the pinned catalog dictionaries and runs them through the full + * pipeline: digest-verified fetch, archive extraction, {@link MecabDictionary} + * load, and {@link LatticeTokenizer} segmentation. Verifies the two catalog + * distributions end to end: IPADIC 2.7.0 (EUC-JP, Japanese) and mecab-ko-dic + * 2.1.1 (UTF-8, Korean). + * + *

Skipped unless {@code -Dopennlp.download.remote=true} is set, matching + * the gate on {@link opennlp.tools.util.DictionaryCatalog}.

+ */ +public class MecabCatalogEndToEndTest { + + @TempDir + private static Path workDir; + + @BeforeAll + static void requireRemoteOptIn() { + Assumptions.assumeTrue(Boolean.getBoolean("opennlp.download.remote"), + "remote downloads are opt-in via -Dopennlp.download.remote=true"); + } + + @Test + void testIpadicInstallsLoadsAndSegments() throws Exception { + MecabDictionary dict = installAndLoad("mecab.ipadic", Charset.forName("EUC-JP")); + LatticeTokenizer tokenizer = new LatticeTokenizer(dict); + + String text = "すもももももももものうち"; + List morphemes = tokenizer.analyze(text); + assertFalse(morphemes.isEmpty()); + assertEquals(List.of("すもも", "も", "もも", "も", "もも", "の", "うち"), + morphemes.stream().map(Morpheme::surface).toList()); + assertCoversText(tokenizer.tokenizePos(text), text); + } + + @Test + void testKoDicInstallsLoadsAndSegments() throws Exception { + MecabDictionary dict = installAndLoad("mecab.ko-dic", StandardCharsets.UTF_8); + LatticeTokenizer tokenizer = new LatticeTokenizer(dict); + + String text = "아버지가 방에 들어가신다"; + List morphemes = tokenizer.analyze(text); + assertFalse(morphemes.isEmpty()); + assertCoversText(tokenizer.tokenizePos(text), text); + } + + /** + * Installs the given catalog dictionary into a fresh directory and loads it. + * + * @param dictionaryId The catalog identifier, for example {@code mecab.ipadic}. + * Must not be null. + * @param charset The encoding the distribution uses. Must not be null. + * @return The loaded dictionary. + */ + private static MecabDictionary installAndLoad(String dictionaryId, Charset charset) + throws Exception { + Path dir = workDir.resolve(dictionaryId); + Files.createDirectories(dir); + int installed = MecabDictionaryInstaller.installFromCatalog(dictionaryId, dir); + assertTrue(installed > 0, "no files extracted for " + dictionaryId); + + try (Stream paths = Files.walk(dir, 3)) { + Path root = paths.filter(p -> p.getFileName().toString().equals("matrix.def")) + .map(Path::getParent) + .findFirst() + .orElseThrow(); + return MecabDictionary.load(root, charset); + } + } + + /** + * Asserts the spans are in order, non-overlapping, and each maps to non-blank text. + * + * @param spans The token spans to check. Must not be null. + * @param text The tokenized text. Must not be null. + */ + private static void assertCoversText(Span[] spans, String text) { + assertTrue(spans.length > 0); + int last = 0; + for (Span span : spans) { + assertTrue(span.getStart() >= last, "spans overlap or regress"); + assertFalse(text.substring(span.getStart(), span.getEnd()).isBlank()); + last = span.getEnd(); + } + } +} From f3e2d3bc03e72cd690573dc57004fcc4e3a86b4a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 16:58:03 -0400 Subject: [PATCH 23/24] OPENNLP-1894: Bound matrix cells separately so real distributions load The cell-count check reused ResourceLimits.MAX_ENTRIES, sized for record-shaped entries, and rejected mecab-ko-dic 2.1.1, whose genuine matrix declares 3822 x 2693 = 10,292,646 cells of two bytes each. A new ResourceLimits.MAX_MATRIX_CELLS bounds two-dimensional cost tables at 2^27 cells (256 MiB of shorts), overridable at startup, and still refuses the roughly 4 GiB allocation a crafted 46340 46340 header would force. Unit tests pin the new bound on both sides with the ko-dic dimensions as the accepted case. --- .../tokenize/lattice/MecabDictionary.java | 9 ++--- .../opennlp/tools/util/ResourceLimits.java | 34 +++++++++++++++--- .../lattice/LatticeTokenizerTest.java | 36 ++++++++++++++----- opennlp-docs/src/docbkx/tokenizer.xml | 5 +-- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java index 8ab1787505..18ac67b48e 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionary.java @@ -54,8 +54,9 @@ * space, so load once and share. Lexicon CSV files under the dictionary directory are * read in sorted path order so tie-breaking is stable across file systems. Connection * costs must cover every declared matrix cell; missing pairs are rejected rather than - * treated as cost zero. Matrix dimensions, the cell count, and the lexicon entry count - * are each bounded by {@link ResourceLimits#MAX_ENTRIES}. Lexicon CSV fields may be + * treated as cost zero. Matrix dimensions and the lexicon entry count are bounded by + * {@link ResourceLimits#MAX_ENTRIES}, and the matrix cell count by + * {@link ResourceLimits#MAX_MATRIX_CELLS}. Lexicon CSV fields may be * MeCab-quoted with {@code ""} escapes. An {@code unk.def} template must name a * category {@code char.def} defined.

* @@ -619,9 +620,9 @@ public static MecabDictionary load(Path directory, Charset charset) throws IOExc throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize + " overflow the addressable connection matrix"); } - if (cells > ResourceLimits.MAX_ENTRIES) { + if (cells > ResourceLimits.MAX_MATRIX_CELLS) { throw new IOException(MATRIX_DEF + " dimensions " + leftSize + " x " + rightSize - + " exceed safe limit of " + ResourceLimits.MAX_ENTRIES); + + " exceed safe limit of " + ResourceLimits.MAX_MATRIX_CELLS); } cellCount = (int) cells; costs = new short[cellCount]; diff --git a/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java b/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java index 5b79fbe955..9cc3771433 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java @@ -35,13 +35,39 @@ public final class ResourceLimits { * (matrix dimensions, lexicon entries, model outcome counts, and similar). * Configurable via {@link #MAX_ENTRIES_PROPERTY}. */ - public static final int MAX_ENTRIES = initMaxEntries(); + public static final int MAX_ENTRIES = initLimit(MAX_ENTRIES_PROPERTY, 10_000_000); + + /** + * System property for overriding {@link #MAX_MATRIX_CELLS}. + * Set at JVM startup, e.g. {@code -DOPENNLP_MAX_MATRIX_CELLS=20000000}. + * Falls back to {@code 134_217_728} if absent or invalid. + */ + public static final String MAX_MATRIX_CELLS_PROPERTY = "OPENNLP_MAX_MATRIX_CELLS"; + + /** + * Upper bound on the cell count of a two-dimensional cost table, whose entries + * are far smaller than the record-sized entries {@link #MAX_ENTRIES} bounds. + * The default of 2^27 cells caps a 16-bit cost matrix at 256 MiB, which admits + * every published MeCab-format distribution (mecab-ko-dic 2.1.1 alone declares + * 3822 x 2693, above {@link #MAX_ENTRIES}) while still refusing the roughly + * 4 GiB allocation a crafted {@code 46340 46340} header would force. + * Configurable via {@link #MAX_MATRIX_CELLS_PROPERTY}. + */ + public static final int MAX_MATRIX_CELLS = + initLimit(MAX_MATRIX_CELLS_PROPERTY, 134_217_728); private ResourceLimits() { } - private static int initMaxEntries() { - final String prop = System.getProperty(MAX_ENTRIES_PROPERTY, "").trim(); + /** + * Reads a positive integer limit from the given system property. + * + * @param property The system property name. Must not be {@code null}. + * @param defaultValue The value used when the property is absent or invalid. + * @return The configured limit, or {@code defaultValue}. + */ + private static int initLimit(String property, int defaultValue) { + final String prop = System.getProperty(property, "").trim(); if (!prop.isEmpty()) { try { final int val = Integer.parseInt(prop); @@ -52,6 +78,6 @@ private static int initMaxEntries() { // Fall through to the default. } } - return 10_000_000; + return defaultValue; } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java index f2acef5238..21525c6073 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/LatticeTokenizerTest.java @@ -760,19 +760,39 @@ void testMatrixDimensionAboveMaxEntriesFailsLoud(@TempDir Path broken) throws IO } /** - * Verifies that a matrix whose cell count is above {@link ResourceLimits#MAX_ENTRIES} - * but still below {@link Integer#MAX_VALUE} is rejected. Without that bound, a header - * such as {@code 46340 46340} would allocate about 4 GiB of shorts. + * Verifies that a matrix whose cell count is above + * {@link ResourceLimits#MAX_MATRIX_CELLS} but still below {@link Integer#MAX_VALUE} + * is rejected. Without that bound, a header such as {@code 46340 46340} would + * allocate about 4 GiB of shorts. */ @Test - void testMatrixCellCountAboveMaxEntriesFailsLoud(@TempDir Path broken) throws IOException { + void testMatrixCellCountAboveMaxCellsFailsLoud(@TempDir Path broken) throws IOException { writeUnitMatrixDictionary(broken); - // 5000 x 5000 = 25_000_000 cells, above the default MAX_ENTRIES of 10_000_000. - write(broken, MATRIX_DEF, "5000 5000\n"); + // 11600 x 11600 = 134_560_000 cells, above the default MAX_MATRIX_CELLS of 2^27. + write(broken, MATRIX_DEF, "11600 11600\n"); final IOException e = Assertions.assertThrows(IOException.class, () -> MecabDictionary.load(broken)); - Assertions.assertEquals("matrix.def dimensions 5000 x 5000 exceed safe limit of " - + ResourceLimits.MAX_ENTRIES, e.getMessage()); + Assertions.assertEquals("matrix.def dimensions 11600 x 11600 exceed safe limit of " + + ResourceLimits.MAX_MATRIX_CELLS, e.getMessage()); + } + + /** + * Verifies that the dimensions of a real published distribution pass the header + * bound. mecab-ko-dic 2.1.1 declares {@code 3822 2693}, which is 10,292,646 cells: + * above {@link ResourceLimits#MAX_ENTRIES} but a legitimate 20 MB cost matrix, so + * the cell bound must be sized to cells rather than reusing the entry bound. The + * load still fails on the truncated body, but with the incomplete-matrix message, + * not the safe-limit one. + */ + @Test + void testKoDicSizedMatrixDimensionsPassTheHeaderBound(@TempDir Path koDic) + throws IOException { + writeUnitMatrixDictionary(koDic); + write(koDic, MATRIX_DEF, "3822 2693\n"); + final IOException e = Assertions.assertThrows(IOException.class, + () -> MecabDictionary.load(koDic)); + Assertions.assertEquals("matrix.def declares 3822 x 2693 connection costs but only 0" + + " pairs are listed", e.getMessage()); } /** diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index a81dea5d3f..7068f31643 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -550,9 +550,10 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { expected SHA-512 digest; the opt-in catalog (installFromCatalog) also needs -Dopennlp.download.remote=true. - matrix.def must list a cost for every declared cell, and matrix + matrix.def must list a cost for every declared cell; matrix dimensions plus lexicon size are bounded by the shared - ResourceLimits.MAX_ENTRIES limit. + ResourceLimits.MAX_ENTRIES limit and the matrix cell count + by ResourceLimits.MAX_MATRIX_CELLS. Extraction caps per-entry size, total bytes, entry count, and gzip expansion; the byte ceilings default to 512 MiB per download and entry and 2 GiB total, and can be raised at JVM startup via From 85cb63f509c5a66cfcf4e83e155451b6157457ee Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 16:58:10 -0400 Subject: [PATCH 24/24] OPENNLP-1894: Extract dictionary files from the archive root only The installer flattened every csv and def file in the archive into the target directory, so mecab-ko-dic's nested user-dic templates, whose numeric fields are empty because they are mecab-dict-index input, landed beside the real lexicon and failed the load. On a case-insensitive file system a template could even overwrite a real lexicon file of the same base name. Entries deeper than one leading directory are now skipped. --- .../lattice/MecabDictionaryInstaller.java | 33 ++++++++++++++++--- .../lattice/MecabDictionaryInstallerTest.java | 23 +++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java index 16e12e3530..8b5589609d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -45,9 +45,12 @@ * names must fit the 100-byte ustar name field. It * extracts only the dictionary payload: the {@code *.csv} lexicon files and * {@code *.def} definition files that a {@link MecabDictionary} reads, plus the - * {@code dicrc} configuration file distributions ship alongside them. Entries are - * flattened to their base names, which also means no archive path can escape the target - * directory.

+ * {@code dicrc} configuration file distributions ship alongside them, taken from the + * archive root only (at most one leading directory deep). Deeper entries are skipped: + * mecab-ko-dic, for example, nests {@code user-dic} templates whose numeric fields are + * empty because they are input for {@code mecab-dict-index}, not loadable lexicon + * data. Extracted entries are flattened to their base names, which also means no + * archive path can escape the target directory.

* *

Extraction is bounded: each entry's declared size, the total bytes written, the * number of extracted dictionary files, and the gzip expansion ratio each have an @@ -280,7 +283,11 @@ static int extract(InputStream archiveStream, Path targetDirectory, long maxEntr } final char type = (char) header[TAR_TYPE_OFFSET]; final String baseName = baseName(name); - final boolean wanted = (type == '0' || type == 0) + // Only the archive root holds dictionary payload. Deeper files such as + // mecab-ko-dic's user-dic templates carry empty numeric fields for + // mecab-dict-index and would fail the load, or on a case-insensitive file + // system overwrite a real lexicon file of the same base name. + final boolean wanted = (type == '0' || type == 0) && pathDepth(name) <= 2 && (baseName.endsWith(".csv") || baseName.endsWith(".def") || "dicrc".equals(baseName)); if (wanted) { @@ -396,6 +403,24 @@ private static String baseName(String name) { return slash < 0 ? name : name.substring(slash + 1); } + /** + * Counts the path segments of a tar entry name, ignoring {@code .} segments and + * empty segments from doubled or trailing slashes. A file at the archive root has + * depth 1 bare or 2 inside the customary versioned top directory. + * + * @param name The tar entry name. + * @return The number of real path segments. + */ + private static int pathDepth(String name) { + int depth = 0; + for (String segment : name.split("/")) { + if (!segment.isEmpty() && !".".equals(segment)) { + depth++; + } + } + return depth; + } + /** * Computes the padding after an entry: tar content is stored in whole blocks. * diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java index 70fdf8048b..e5450ff3b1 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstallerTest.java @@ -62,6 +62,29 @@ void testExtractsDictionaryFilesAndFlattensPaths(@TempDir Path target) Files.readString(target.resolve("lexicon.csv"))); } + /** + * Verifies that only files at the archive root count as dictionary payload. + * mecab-ko-dic ships template user dictionaries under {@code user-dic/} whose + * numeric fields are empty, input for {@code mecab-dict-index} rather than loadable + * lexicon data. Flattening them next to the real lexicon fails the subsequent load, + * and on a case-insensitive file system a template can silently overwrite a real + * lexicon file of the same base name. + */ + @Test + void testNestedTemplateFilesAreNotExtracted(@TempDir Path target) throws IOException { + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { + {"dict-1.0/NNP.csv", "cat,1786,3546,2953,noun\n"}, + {"dict-1.0/matrix.def", "1 1\n0 0 0\n"}, + {"dict-1.0/user-dic/person.csv", "template,,,,noun\n"}}); + + final int extracted = MecabDictionaryInstaller.extract( + new ByteArrayInputStream(archive), target); + + Assertions.assertEquals(2, extracted); + Assertions.assertTrue(Files.exists(target.resolve("NNP.csv"))); + Assertions.assertTrue(Files.notExists(target.resolve("person.csv"))); + } + @Test void testInstallReadsAFileUri(@TempDir Path source, @TempDir Path target) throws IOException {