Unknown text is handled through the dictionary's character categories: where the + * lexicon has no entry, or a category always invokes them, unknown-word candidates are + * generated per category template, grouping runs of same-category characters when the + * category says so. Whitespace never joins a morpheme and is never reported as one. + * Every reported span is in original text coordinates.
+ * + *{@link #analyze(String)} returns full morphemes with their dictionary features; + * the {@link Tokenizer} view reports just the surfaces and spans.
+ * + *The tokenizer reads only immutable dictionary state and is safe to share between + * threads.
+ * + * @since 3.0.0 + */ +public class LatticeTokenizer implements Tokenizer { + + /** The context id of the beginning and end of text. */ + private static final int BOUNDARY_CONTEXT = 0; + + private final MecabDictionary dictionary; + + /** + * Initializes the tokenizer. + * + * @param dictionary The dictionary to segment with. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public LatticeTokenizer(MecabDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + /** + * One lattice node: a candidate morpheme with its best path cost so far. Nodes + * ending at one position chain through {@link #nextEndingHere}. + */ + private static final class Node { + private final int start; + private final int end; + private final WordEntry entry; + private final boolean unknown; + private long pathCost = Long.MAX_VALUE; + private Node previous; + private Node nextEndingHere; + + private Node(int start, int end, WordEntry entry, boolean unknown) { + this.start = start; + this.end = end; + this.entry = entry; + this.unknown = unknown; + } + } + + /** + * Segments a text into morphemes with their dictionary features. + * + * @param text The text to segment. Must not be {@code null}. + * @return The morphemes in text order, spans in original coordinates, whitespace + * omitted. Never {@code null}; empty for empty or all-whitespace input. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position, which a {@code unk.def} without a {@code DEFAULT} template does. + */ + public ListReports the segmented surfaces, whitespace omitted.
+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. + */ + @Override + public String[] tokenize(String text) { + final ListReports the segmented spans in original text coordinates, whitespace omitted.
+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if the dictionary offers no candidate at some + * position; see {@link #analyze(String)}. + */ + @Override + public Span[] tokenizePos(String text) { + final ListEvery candidate stays inside the same-category run, so an unknown word never + * glues characters of different categories together, and every length counts whole + * characters rather than code units.
+ * + * @param candidates Receives the candidates. + * @param text The text being segmented. + * @param position The position the candidates start at. + * @param runEnd The exclusive end of the same-category run starting at + * {@code position}. + * @param category The category of that run. + * @param templates The category's unknown-word templates. + */ + private void addUnknown(ListThe same format serves multiple languages: the Japanese + * IPADIC and + * UniDic distributions and the Korean + * mecab-ko-dic all load + * through this one reader, with the feature columns passed through untouched because + * their schemas differ.
+ * + *Each instance keeps about 0.75 MB of category tables keyed by the 16-bit code-unit + * space, so load once and share. Lexicon CSV files under the dictionary directory are + * read in sorted path order so tie-breaking is stable across file systems. Connection + * costs must cover every declared matrix cell; missing pairs are rejected rather than + * treated as cost zero. Matrix dimensions and the lexicon entry count are bounded by + * {@link ResourceLimits#MAX_ENTRIES}, and the matrix cell count by + * {@link ResourceLimits#MAX_MATRIX_CELLS}. Lexicon CSV fields may be + * MeCab-quoted with {@code ""} escapes. An {@code unk.def} template must name a + * category {@code char.def} defined.
+ * + *Instances are immutable and safe to share between threads.
+ * + * @see LatticeTokenizer + * @since 3.0.0 + */ +public final class MecabDictionary { + + /** + * The category name every {@code char.def} must define; unmapped code points and + * unknown-word handling fall back to it. + */ + static final String DEFAULT_CATEGORY = "DEFAULT"; + + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** The prefix a {@code char.def} code point field carries, in either letter case. */ + private static final String HEX_PREFIX = "0x"; + + /** The separator between the two ends of a {@code char.def} code point range. */ + private static final String RANGE_SEPARATOR = ".."; + + /** The {@code char.def} field value that turns a category flag on. */ + private static final String FLAG_ON = "1"; + + /** The {@code char.def} field value that turns a category flag off. */ + private static final String FLAG_OFF = "0"; + + /** + * One lexicon or unknown-word entry. + * + * @param leftId The left context id, an index into the connection matrix. + * @param rightId The right context id, an index into the connection matrix. + * @param cost The entry's own cost. + * @param features The entry's feature columns, in file order. + */ + record WordEntry(int leftId, int rightId, int cost, ListThe layout is the classic base/check pair: from state {@code s}, label + * {@code c} leads to {@code t = base[s] + c} exactly when {@code check[t] == s}. + * Label {@code 0} terminates a surface and leads to a state whose negative base + * encodes the index of the surface's entry list.
+ */ + private static final class DoubleArrayLexicon { + + private final int[] base; + private final int[] check; + private final int[] codeOf; + private final ListThe Basic Multilingual Plane is held in a directly indexed array. The + * supplementary planes are held as a sorted, non-overlapping range table searched by + * binary search, because dictionaries map them in a handful of large blocks.
+ */ + private static final class CategoryTable { + + private final Category[] bmp; + private final int[] rangeStart; + private final int[] rangeEnd; + private final Category[] rangeCategory; + + private CategoryTable(Category[] bmp, int[] rangeStart, int[] rangeEnd, + Category[] rangeCategory) { + this.bmp = bmp; + this.rangeStart = rangeStart; + this.rangeEnd = rangeEnd; + this.rangeCategory = rangeCategory; + } + + /** + * Looks up the category a {@code char.def} mapping gives a code point. The table + * holds the {@link Category} instances themselves, and two code points of one + * category share one instance, so categories may be compared by identity. + * + * @param codePoint The code point to classify. + * @return The category, or {@code null} when no mapping covers the code point. + */ + private Category categoryOf(int codePoint) { + if (codePoint <= Character.MAX_VALUE) { + return bmp[codePoint]; + } + int low = 0; + int high = rangeStart.length - 1; + while (low <= high) { + final int middle = (low + high) >>> 1; + if (codePoint < rangeStart[middle]) { + high = middle - 1; + } else if (codePoint > rangeEnd[middle]) { + low = middle + 1; + } else { + return rangeCategory[middle]; + } + } + return null; + } + } + + /** + * Collects {@code char.def} mappings in file order and folds them into a + * {@link CategoryTable}, giving a later mapping precedence over an earlier one that + * covers the same code point, which is what direct indexing does for the BMP. + */ + private static final class CategoryTableBuilder { + + private final String[] bmp = new String[Character.MAX_VALUE + 1]; + private final ListThe features are the entry's columns exactly as listed in the dictionary, since + * different dictionaries carry different schemas: part of speech first by convention, + * then dictionary-specific columns such as conjugation, base form, or reading. A + * morpheme produced by unknown-word handling has the unknown entry's features and is + * marked as such.
+ * + * @param span The location of the morpheme in the original text. Must not be + * {@code null}. + * @param surface The covered text. Must not be {@code null} or empty. + * @param features The dictionary feature columns. Must not be {@code null}. + * @param unknown Whether the morpheme came from unknown-word handling rather than a + * lexicon entry. + * + * @since 3.0.0 + */ +public record Morpheme(Span span, String surface, ListThe lexicon format is one entry per line: the word, its count, and optionally a + * tag, separated by whitespace. The lexicon file is user-supplied; no lexicon data is + * bundled. Every reported span is in original text coordinates.
+ * + *Instances are immutable and safe to share between threads.
+ * + * @since 3.0.0 + */ +public class UnigramSegmenter implements Tokenizer { + + /** The log-probability charged to a character the lexicon does not know. */ + private final double unknownLogProbability; + + private final WordTrie trie; + + /** + * One immutable trie node: children are a sorted character array with a parallel + * node array, found by binary search, so a descent never boxes a {@link Character}. + */ + private static final class WordTrie { + + private final char[] keys; + private final WordTrie[] nodes; + private final double logProbability; + + private WordTrie(char[] keys, WordTrie[] nodes, double logProbability) { + this.keys = keys; + this.nodes = nodes; + this.logProbability = logProbability; + } + + /** + * Descends one character. + * + * @param c The next surface character. + * @return The child node, or {@code null} when no surface continues with {@code c}. + */ + private WordTrie child(char c) { + final int index = Arrays.binarySearch(keys, c); + return index >= 0 ? nodes[index] : null; + } + } + + /** One mutable trie node during construction, copied into a {@link WordTrie}. */ + private static final class WordTrieBuilder { + + private final MapReports the segmented surfaces, whitespace omitted.
+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public String[] tokenize(String text) { + return Span.spansToStrings(tokenizePos(text), text); + } + + /** + * {@inheritDoc} + * + *Reports the segmented spans in original text coordinates, whitespace omitted.
+ * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public Span[] tokenizePos(String text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + final List spans = new ArrayList<>(); + int start = 0; + while (start < text.length()) { + if (StringUtil.isWhitespace(text.charAt(start))) { + start++; + continue; + } + int end = start; + while (end < text.length() && !StringUtil.isWhitespace(text.charAt(end))) { + end++; + } + decode(text, start, end, spans); + start = end; + } + return spans.toArray(new Span[0]); + } + + /** + * Viterbi over word log-probabilities within one whitespace-free stretch. + * + * @param text The text being segmented. + * @param from The stretch start. + * @param to The exclusive stretch end. + * @param spans Receives the best path's spans, in text order and in original text + * coordinates. + */ + private void decode(String text, int from, int to, List spans) { + final int length = to - from; + final double[] best = new double[length + 1]; + final int[] previous = new int[length + 1]; + for (int i = 1; i <= length; i++) { + best[i] = Double.NEGATIVE_INFINITY; + } + for (int i = 0; i < length; i++) { + if (best[i] == Double.NEGATIVE_INFINITY) { + continue; + } + // A single-character step at the unknown log-probability keeps every position + // reachable even where no lexicon word matches. The step advances one code + // point, never one code unit, so an unknown supplementary character is stepped + // over whole and no span boundary can land between its surrogate halves. + final int width = Character.charCount(text.codePointAt(from + i)); + final double fallback = best[i] + unknownLogProbability; + if (i + width <= length && fallback > best[i + width]) { + best[i + width] = fallback; + previous[i + width] = i; + } + WordTrie node = trie; + for (int j = from + i; j < to; j++) { + node = node.child(text.charAt(j)); + if (node == null) { + break; + } + if (!Double.isNaN(node.logProbability)) { + final int end = j - from + 1; + final double score = best[i] + node.logProbability; + if (score > best[end]) { + best[end] = score; + previous[end] = i; + } + } + } + } + final List reversed = new ArrayList<>(); + for (int end = length; end > 0; end = previous[end]) { + reversed.add(new Span(from + previous[end], from + end)); + } + for (int i = reversed.size() - 1; i >= 0; i--) { + spans.add(reversed.get(i)); + } + } + + /** + * Finds the first whitespace character in a lexicon line. + * + * @param text The line to scan. + * @return The index of the first whitespace character, or {@code -1} when the line + * contains none. + */ + private static int whitespaceIndex(String text) { + for (int i = 0; i < text.length(); i++) { + if (StringUtil.isWhitespace(text.charAt(i))) { + return i; + } + } + return -1; + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java b/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java new file mode 100644 index 0000000000..9cc3771433 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/util/ResourceLimits.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +/** + * Shared upper bounds for counts read from user-supplied resources, so a crafted + * file cannot force an outsized allocation before validation completes. + */ +public final class ResourceLimits { + + /** + * System property for overriding {@link #MAX_ENTRIES}. + * Set at JVM startup, e.g. {@code -DOPENNLP_MAX_ENTRIES=5000000}. + * Falls back to {@code 10_000_000} if absent or invalid. + */ + public static final String MAX_ENTRIES_PROPERTY = "OPENNLP_MAX_ENTRIES"; + + /** + * Upper bound on count fields and resource sizes that drive allocations + * (matrix dimensions, lexicon entries, model outcome counts, and similar). + * Configurable via {@link #MAX_ENTRIES_PROPERTY}. + */ + public static final int MAX_ENTRIES = initLimit(MAX_ENTRIES_PROPERTY, 10_000_000); + + /** + * System property for overriding {@link #MAX_MATRIX_CELLS}. + * Set at JVM startup, e.g. {@code -DOPENNLP_MAX_MATRIX_CELLS=20000000}. + * Falls back to {@code 134_217_728} if absent or invalid. + */ + public static final String MAX_MATRIX_CELLS_PROPERTY = "OPENNLP_MAX_MATRIX_CELLS"; + + /** + * Upper bound on the cell count of a two-dimensional cost table, whose entries + * are far smaller than the record-sized entries {@link #MAX_ENTRIES} bounds. + * The default of 2^27 cells caps a 16-bit cost matrix at 256 MiB, which admits + * every published MeCab-format distribution (mecab-ko-dic 2.1.1 alone declares + * 3822 x 2693, above {@link #MAX_ENTRIES}) while still refusing the roughly + * 4 GiB allocation a crafted {@code 46340 46340} header would force. + * Configurable via {@link #MAX_MATRIX_CELLS_PROPERTY}. + */ + public static final int MAX_MATRIX_CELLS = + initLimit(MAX_MATRIX_CELLS_PROPERTY, 134_217_728); + + private ResourceLimits() { + } + + /** + * Reads a positive integer limit from the given system property. + * + * @param property The system property name. Must not be {@code null}. + * @param defaultValue The value used when the property is absent or invalid. + * @return The configured limit, or {@code defaultValue}. + */ + private static int initLimit(String property, int defaultValue) { + final String prop = System.getProperty(property, "").trim(); + if (!prop.isEmpty()) { + try { + final int val = Integer.parseInt(prop); + if (val > 0) { + return val; + } + } catch (NumberFormatException ignore) { + // Fall through to the default. + } + } + return defaultValue; + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java index 8325b04f21..c24349401f 100644 --- a/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java +++ b/opennlp-core/opennlp-ml/opennlp-ml-commons/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java @@ -24,6 +24,8 @@ import java.util.StringTokenizer; import java.util.zip.GZIPInputStream; +import opennlp.tools.util.ResourceLimits; + /** * An abstract, basic implementation of a model reader. */ @@ -32,31 +34,17 @@ public abstract class AbstractModelReader { /** * System property for overriding the maximum number of entries (outcomes, predicates, * outcome patterns, chunk counts) that may be read from a model file or training data. - * Set at JVM startup, e.g. {@code -DOPENNLP_MAX_ENTRIES=5000000}. - * Falls back to {@code 10_000_000} if absent or invalid. + * Alias of {@link ResourceLimits#MAX_ENTRIES_PROPERTY}. */ - public static final String MAX_ENTRIES_PROPERTY = "OPENNLP_MAX_ENTRIES"; + public static final String MAX_ENTRIES_PROPERTY = ResourceLimits.MAX_ENTRIES_PROPERTY; /** * Upper bound on count fields read from a model file. - * Prevents OOM on crafted inputs with oversized array size declarations. - * Configurable via the {@link #MAX_ENTRIES_PROPERTY} system property. - *+ * Alias of {@link ResourceLimits#MAX_ENTRIES}. * Public so that deserializers outside this package which implement their own binary * format can apply the same bound to their count fields. */ - public static final int MAX_ENTRIES = initMaxEntries(); - - private static int initMaxEntries() { - String prop = System.getProperty(MAX_ENTRIES_PROPERTY, "").trim(); - if (!prop.isEmpty()) { - try { - int val = Integer.parseInt(prop); - if (val > 0) return val; - } catch (NumberFormatException ignore) { } - } - return 10_000_000; - } + public static final int MAX_ENTRIES = ResourceLimits.MAX_ENTRIES; /** * The number of predicates contained in a model. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java new file mode 100644 index 0000000000..8b5589609d --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/tokenize/lattice/MecabDictionaryInstaller.java @@ -0,0 +1,574 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.tokenize.lattice; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.zip.GZIPInputStream; + +import opennlp.tools.util.DictionaryCatalog; +import opennlp.tools.util.DownloadUtil; +import opennlp.tools.util.model.UncloseableInputStream; + +/** + * Fetches and unpacks a MeCab-format dictionary archive into a local directory, so the + * dictionary is acquired by the user at install time and never ships with this library. + * No dictionary data is bundled. Any non-{@code file:} archive is downloaded through + * {@link DownloadUtil#download(URI, Path, String)} and requires an expected SHA-512 + * digest. Built-in catalog URLs are opt-in via {@link #installFromCatalog(String, Path)}. + * + *
The installer reads gzip-compressed + * + * ustar archives (POSIX.1-1988), the format the common distributions use. GNU + * long-name ({@code L}) and PAX ({@code x}/{@code g}) headers are not supported; entry + * names must fit the 100-byte ustar name field. It + * extracts only the dictionary payload: the {@code *.csv} lexicon files and + * {@code *.def} definition files that a {@link MecabDictionary} reads, plus the + * {@code dicrc} configuration file distributions ship alongside them, taken from the + * archive root only (at most one leading directory deep). Deeper entries are skipped: + * mecab-ko-dic, for example, nests {@code user-dic} templates whose numeric fields are + * empty because they are input for {@code mecab-dict-index}, not loadable lexicon + * data. Extracted entries are flattened to their base names, which also means no + * archive path can escape the target directory.
+ * + *Extraction is bounded: each entry's declared size, the total bytes written, the + * number of extracted dictionary files, and the gzip expansion ratio each have an + * explicit ceiling so a crafted archive cannot fill the disk. The byte ceilings can be + * raised at JVM startup via {@link #MAX_ENTRY_BYTES_PROPERTY} and + * {@link #MAX_TOTAL_EXTRACTED_BYTES_PROPERTY} for dictionaries larger than the + * defaults, such as UniDic.
+ * + * @since 3.0.0 + */ +public final class MecabDictionaryInstaller { + + private static final int TAR_BLOCK = 512; + private static final int TAR_NAME_LENGTH = 100; + private static final int TAR_SIZE_OFFSET = 124; + private static final int TAR_SIZE_LENGTH = 12; + private static final int TAR_TYPE_OFFSET = 156; + + /** + * System property for overriding {@link #MAX_ENTRY_BYTES}. Set at JVM startup, + * e.g. {@code -Dopennlp.install.max.entry.bytes=2147483648} for dictionaries whose + * lexicon files exceed the default ceiling. Falls back to the default if absent, + * non-numeric, or not positive. + */ + public static final String MAX_ENTRY_BYTES_PROPERTY = "opennlp.install.max.entry.bytes"; + + /** + * System property for overriding {@link #MAX_TOTAL_EXTRACTED_BYTES}. Set at JVM + * startup, e.g. {@code -Dopennlp.install.max.total.bytes=8589934592}. Falls back to + * the default if absent, non-numeric, or not positive. + */ + public static final String MAX_TOTAL_EXTRACTED_BYTES_PROPERTY = + "opennlp.install.max.total.bytes"; + + /** + * Inclusive ceiling on one tar entry's declared size, in bytes: 512 MiB unless + * overridden via {@link #MAX_ENTRY_BYTES_PROPERTY}. + */ + static final long MAX_ENTRY_BYTES = + DownloadUtil.configuredLimit(MAX_ENTRY_BYTES_PROPERTY, 512L * 1024 * 1024); + + /** + * Inclusive ceiling on the sum of extracted dictionary file sizes, in bytes: 2 GiB + * unless overridden via {@link #MAX_TOTAL_EXTRACTED_BYTES_PROPERTY}. + */ + static final long MAX_TOTAL_EXTRACTED_BYTES = + DownloadUtil.configuredLimit(MAX_TOTAL_EXTRACTED_BYTES_PROPERTY, + 2L * 1024 * 1024 * 1024); + + /** Inclusive ceiling on the number of dictionary files extracted from one archive. */ + static final int MAX_EXTRACTED_ENTRIES = 10_000; + + /** + * Inclusive ceiling on decompressed bytes per compressed byte while reading the + * gzip wrapper; higher expansion fails before the payload is written. + */ + static final int MAX_GZIP_EXPANSION_RATIO = 100; + + private MecabDictionaryInstaller() { + // This class exposes only static methods and is never instantiated. + } + + /** + * Unpacks a local {@code file:} archive URI. Any other scheme requires + * {@link #install(URI, Path, String)} with an expected SHA-512 digest. + * + * @param archive The archive location, a gzip-compressed ustar tar. Must not be + * {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if reading or writing fails, the archive contains no + * dictionary file, or an extraction budget is exceeded. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code archive} is not an absolute URI, or {@code archive} is not a + * {@code file:} URI. + */ + public static int install(URI archive, Path targetDirectory) throws IOException { + return install(archive, targetDirectory, null); + } + + /** + * Downloads a dictionary archive when needed, verifies its SHA-512 digest through + * {@link DownloadUtil#download(URI, Path, String)}, and unpacks it. A {@code file:} + * URI may omit the digest and is then opened without verification. + * + * @param archive The archive location, a gzip-compressed ustar tar. Must not be + * {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @param expectedSha512 The expected SHA-512 hex digest. Required for any + * non-{@code file:} URI; optional for {@code file:} URIs. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if fetching, verification, reading, or writing fails, + * the archive contains no dictionary file, or an extraction budget is exceeded. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code archive} is not an absolute URI, or a non-{@code file:} URI omits + * the digest. + */ + public static int install(URI archive, Path targetDirectory, String expectedSha512) + throws IOException { + if (archive == null) { + throw new IllegalArgumentException("archive must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); + } + if (!archive.isAbsolute()) { + throw new IllegalArgumentException("archive must be an absolute URI"); + } + if (expectedSha512 == null) { + if (!isLocalFile(archive)) { + throw new IllegalArgumentException("a non-file archive requires an expected " + + "SHA-512 digest; use install(URI, Path, String)"); + } + try (InputStream in = archive.toURL().openStream()) { + return extract(in, targetDirectory); + } + } + final Path downloaded = Files.createTempFile("mecab-dict-", ".tar.gz"); + try { + DownloadUtil.download(archive, downloaded, expectedSha512); + try (InputStream in = Files.newInputStream(downloaded)) { + return extract(in, targetDirectory); + } + } finally { + Files.deleteIfExists(downloaded); + } + } + + /** + * Downloads a dictionary named in {@link DictionaryCatalog} and unpacks it. Requires + * {@code -Dopennlp.download.remote=true}. + * + * @param dictionaryId The catalog id, for example {@code mecab.ipadic} or + * {@code mecab.ko-dic}. Must not be {@code null}. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if the catalog entry is missing, remote downloads are + * disabled, or install fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int installFromCatalog(String dictionaryId, Path targetDirectory) + throws IOException { + if (dictionaryId == null) { + throw new IllegalArgumentException("dictionaryId must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); + } + final Path downloaded = Files.createTempFile("mecab-dict-", ".tar.gz"); + try { + DictionaryCatalog.loadDefault().download(dictionaryId, downloaded); + return install(downloaded.toUri(), targetDirectory); + } finally { + Files.deleteIfExists(downloaded); + } + } + + /** + * {@return {@code true} when {@code archive} uses the {@code file} scheme} + * + * @param archive The absolute archive URI. + */ + private static boolean isLocalFile(URI archive) { + return "file".equalsIgnoreCase(archive.getScheme()); + } + + /** + * Unpacks a dictionary archive stream under the production extraction budgets. + * + * @param archiveStream The gzip-compressed ustar tar content. Must not be + * {@code null}. Not closed. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if reading or writing fails, the archive contains no + * dictionary file, or an extraction budget is exceeded. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static int extract(InputStream archiveStream, Path targetDirectory) + throws IOException { + return extract(archiveStream, targetDirectory, MAX_ENTRY_BYTES, + MAX_TOTAL_EXTRACTED_BYTES, MAX_EXTRACTED_ENTRIES, MAX_GZIP_EXPANSION_RATIO); + } + + /** + * Unpacks a dictionary archive stream under caller-supplied budgets. + * + * @param archiveStream The gzip-compressed ustar tar content. Must not be + * {@code null}. Not closed. + * @param targetDirectory The directory to unpack into; created when absent. Must not + * be {@code null}. + * @param maxEntryBytes Inclusive ceiling on one entry's declared size. + * @param maxTotalBytes Inclusive ceiling on total extracted bytes. + * @param maxEntries Inclusive ceiling on extracted dictionary file count. + * @param maxGzipRatio Inclusive ceiling on decompressed bytes per compressed byte. + * @return The number of dictionary files extracted. + * @throws IOException Thrown if reading or writing fails, the archive contains no + * dictionary file, or a budget is exceeded. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + static int extract(InputStream archiveStream, Path targetDirectory, long maxEntryBytes, + long maxTotalBytes, int maxEntries, int maxGzipRatio) throws IOException { + if (archiveStream == null) { + throw new IllegalArgumentException("archiveStream must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); + } + Files.createDirectories(targetDirectory); + final CountingInputStream compressed = new CountingInputStream(archiveStream); + try (GZIPInputStream gzip = new GZIPInputStream( + new UncloseableInputStream(compressed))) { + final BudgetedInputStream tar = + new BudgetedInputStream(gzip, compressed, maxGzipRatio); + final byte[] header = new byte[TAR_BLOCK]; + int extracted = 0; + long totalExtracted = 0; + while (readBlock(tar, header)) { + if (isEndBlock(header)) { + break; + } + final String name = headerName(header); + final long size = headerSize(header); + if (size > maxEntryBytes) { + throw new IOException( + "tar entry size exceeds safe limit of " + maxEntryBytes); + } + final char type = (char) header[TAR_TYPE_OFFSET]; + final String baseName = baseName(name); + // Only the archive root holds dictionary payload. Deeper files such as + // mecab-ko-dic's user-dic templates carry empty numeric fields for + // mecab-dict-index and would fail the load, or on a case-insensitive file + // system overwrite a real lexicon file of the same base name. + final boolean wanted = (type == '0' || type == 0) && pathDepth(name) <= 2 + && (baseName.endsWith(".csv") || baseName.endsWith(".def") + || "dicrc".equals(baseName)); + if (wanted) { + if (extracted >= maxEntries) { + throw new IOException( + "extracted entry count exceeds safe limit of " + maxEntries); + } + if (totalExtracted + size > maxTotalBytes) { + throw new IOException( + "extracted archive size exceeds safe limit of " + maxTotalBytes); + } + final Path file = targetDirectory.resolve(baseName); + try (InputStream entry = boundedStream(tar, size)) { + Files.copy(entry, file, StandardCopyOption.REPLACE_EXISTING); + } + extracted++; + totalExtracted += size; + skip(tar, padding(size)); + } else { + skip(tar, size + padding(size)); + } + } + if (extracted == 0) { + throw new IOException("the archive contains no dictionary file"); + } + return extracted; + } + } + + /** + * Fills one tar block from the stream. + * + * @param in The tar stream. + * @param block The block buffer to fill completely. + * @return {@code true} when a full block was read, {@code false} at a clean end of + * stream before any byte of the block. + * @throws IOException Thrown if the stream ends inside the block or reading fails. + */ + private static boolean readBlock(InputStream in, byte[] block) throws IOException { + int filled = 0; + while (filled < block.length) { + final int read = in.read(block, filled, block.length - filled); + if (read < 0) { + if (filled == 0) { + return false; + } + throw new IOException("truncated tar header"); + } + filled += read; + } + return true; + } + + /** + * Recognizes the all-zero block that terminates a tar archive. + * + * @param block The block to inspect. + * @return {@code true} when every byte is zero. + */ + private static boolean isEndBlock(byte[] block) { + for (final byte b : block) { + if (b != 0) { + return false; + } + } + return true; + } + + /** + * Reads the NUL-terminated entry name from a tar header block. + * + * @param header The header block. + * @return The entry name. Never {@code null}. + */ + private static String headerName(byte[] header) { + int end = 0; + while (end < TAR_NAME_LENGTH && header[end] != 0) { + end++; + } + return new String(header, 0, end, StandardCharsets.UTF_8); + } + + /** + * Reads the octal entry size from a tar header block. + * + * @param header The header block. + * @return The entry size in bytes. + * @throws IOException Thrown if the size field holds a non-octal digit. + */ + private static long headerSize(byte[] header) throws IOException { + long size = 0; + for (int i = TAR_SIZE_OFFSET; i < TAR_SIZE_OFFSET + TAR_SIZE_LENGTH; i++) { + final byte b = header[i]; + if (b == 0 || b == ' ') { + continue; + } + if (b < '0' || b > '7') { + throw new IOException("malformed tar size field"); + } + size = size * 8 + (b - '0'); + } + return size; + } + + /** + * Strips any directory prefix from an archive entry name. + * + * @param name The entry name as stored in the archive. + * @return The part after the last {@code /}, or the whole name when there is none. + */ + private static String baseName(String name) { + final int slash = name.lastIndexOf('/'); + return slash < 0 ? name : name.substring(slash + 1); + } + + /** + * Counts the path segments of a tar entry name, ignoring {@code .} segments and + * empty segments from doubled or trailing slashes. A file at the archive root has + * depth 1 bare or 2 inside the customary versioned top directory. + * + * @param name The tar entry name. + * @return The number of real path segments. + */ + private static int pathDepth(String name) { + int depth = 0; + for (String segment : name.split("/")) { + if (!segment.isEmpty() && !".".equals(segment)) { + depth++; + } + } + return depth; + } + + /** + * Computes the padding after an entry: tar content is stored in whole blocks. + * + * @param size The entry size in bytes. + * @return The number of padding bytes up to the next block boundary. + */ + private static long padding(long size) { + final long remainder = size % TAR_BLOCK; + return remainder == 0 ? 0 : TAR_BLOCK - remainder; + } + + /** + * Consumes and discards an exact number of bytes from the stream. + * + * @param in The stream to read from. + * @param bytes The number of bytes to discard. + * @throws IOException Thrown if the stream ends before that many bytes were read. + */ + private static void skip(InputStream in, long bytes) throws IOException { + long remaining = bytes; + final byte[] buffer = new byte[8192]; + while (remaining > 0) { + final int read = in.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + } + } + + /** + * Wraps the tar stream so exactly one entry's bytes are readable. + * + * @param in The tar stream, positioned at the entry's first byte. + * @param size The entry size in bytes. + * @return A stream reporting end of stream after that many bytes, and failing if the + * tar stream ends first. Never {@code null}; closing it leaves {@code in} + * open and positioned after the entry content. + */ + private static InputStream boundedStream(InputStream in, long size) { + return new InputStream() { + private long remaining = size; + + @Override + public int read() throws IOException { + if (remaining <= 0) { + return -1; + } + final int b = in.read(); + if (b < 0) { + throw new IOException("truncated tar entry"); + } + remaining--; + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + if (remaining <= 0) { + return -1; + } + final int read = in.read(buffer, offset, (int) Math.min(length, remaining)); + if (read < 0) { + throw new IOException("truncated tar entry"); + } + remaining -= read; + return read; + } + }; + } + + /** + * Counts bytes read from a delegate stream. + */ + private static final class CountingInputStream extends FilterInputStream { + + private long count; + + private CountingInputStream(InputStream in) { + super(in); + } + + private long count() { + return count; + } + + @Override + public int read() throws IOException { + final int b = super.read(); + if (b >= 0) { + count++; + } + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + final int read = super.read(buffer, offset, length); + if (read > 0) { + count += read; + } + return read; + } + } + + /** + * Counts decompressed bytes and rejects a gzip expansion above the supplied ratio. + */ + private static final class BudgetedInputStream extends FilterInputStream { + + private final CountingInputStream compressed; + private final int maxGzipRatio; + private long decompressed; + + private BudgetedInputStream(InputStream in, CountingInputStream compressed, + int maxGzipRatio) { + super(in); + this.compressed = compressed; + this.maxGzipRatio = maxGzipRatio; + } + + @Override + public int read() throws IOException { + final int b = super.read(); + if (b >= 0) { + decompressed++; + checkRatio(); + } + return b; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + final int read = super.read(buffer, offset, length); + if (read > 0) { + decompressed += read; + checkRatio(); + } + return read; + } + + private void checkRatio() throws IOException { + final long compressedBytes = compressed.count(); + if (compressedBytes > 0 + && decompressed > (long) maxGzipRatio * compressedBytes) { + throw new IOException( + "gzip expansion ratio exceeds safe limit of " + maxGzipRatio); + } + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java new file mode 100644 index 0000000000..5e32953346 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Properties; +import java.util.Set; + +/** + * Opt-in catalog of remote dictionary archives and companion files. The catalog + * ships URLs and SHA-512 digests only; it never bundles the data itself. Fetching + * an entry requires {@link DownloadUtil#REMOTE_DOWNLOAD_PROPERTY} to be + * {@code true}, so enabling a built-in URL is an explicit user action. + * + * @since 3.0.0 + */ +public final class DictionaryCatalog { + + private static final String DEFAULT_RESOURCE = + "opennlp/tools/util/dictionary-catalog.properties"; + + private final Properties properties; + + private DictionaryCatalog(Properties properties) { + this.properties = properties; + } + + /** + * Loads the catalog shipped on the classpath. + * + * @return The catalog. Never {@code null}. + * @throws IOException Thrown if the resource is missing or cannot be read. + */ + public static DictionaryCatalog loadDefault() throws IOException { + try (InputStream in = DictionaryCatalog.class.getClassLoader() + .getResourceAsStream(DEFAULT_RESOURCE)) { + if (in == null) { + throw new IOException("missing classpath resource " + DEFAULT_RESOURCE); + } + return load(in); + } + } + + /** + * Loads a catalog from a properties stream. + * + * @param in The properties content. Must not be {@code null}. + * @return The catalog. Never {@code null}. + * @throws IOException Thrown if reading fails. + * @throws IllegalArgumentException Thrown if {@code in} is {@code null}. + */ + public static DictionaryCatalog load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + final Properties properties = new Properties(); + properties.load(in); + return new DictionaryCatalog(properties); + } + + /** + * {@return the catalog entry ids, in encounter order} + */ + public SetSource strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Japanese dictionary as the sibling usage + * example, whose javadoc spells out each fixture word.
+ */ +public class LatticeTokenizerTest { + + private static final String LEXICON_CSV = "lexicon.csv"; + private static final String MATRIX_DEF = "matrix.def"; + private static final String CHAR_DEF = "char.def"; + private static final String UNK_DEF = "unk.def"; + + /** A one by one connection matrix charging cost zero, for single-context fixtures. */ + private static final String UNIT_MATRIX = "1 1\n0 0 0\n"; + + /** + * The {@code char.def} line defining the DEFAULT category: it does not invoke + * unknown-word handling beside a lexicon match, it groups a whole run into one + * candidate, and it offers no fixed-length candidates. + */ + private static final String DEFAULT_CATEGORY_LINE = "DEFAULT 0 1 0"; + + /** The {@code unk.def} template line for the DEFAULT category. */ + private static final String DEFAULT_UNKNOWN_TEMPLATE = "DEFAULT,0,0,10000,symbol,unknown"; + + @TempDir + static Path directory; + + private static LatticeTokenizer tokenizer; + + @BeforeAll + static void loadDictionary() throws IOException { + write(LEXICON_CSV, String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "")); + write(MATRIX_DEF, UNIT_MATRIX); + write(CHAR_DEF, String.join("\n", + DEFAULT_CATEGORY_LINE, + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "LATIN 1 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "0x0041..0x005A LATIN", + "0x0061..0x007A LATIN", + "")); + write(UNK_DEF, String.join("\n", + DEFAULT_UNKNOWN_TEMPLATE, + "LATIN,0,0,4000,noun,foreign", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")); + tokenizer = new LatticeTokenizer(MecabDictionary.load(directory)); + } + + /** Writes one UTF-8 dictionary file into the shared dictionary directory. */ + private static void write(String name, String content) throws IOException { + write(directory, name, content); + } + + /** Writes one UTF-8 dictionary file into a test-supplied directory. */ + private static void write(Path target, String name, String content) throws IOException { + Files.write(target.resolve(name), content.getBytes(StandardCharsets.UTF_8)); + } + + @Test + void testLatticePrefersTheCheaperSegmentation() { + // Tokyo plus the metropolis suffix must beat the competing reading east plus Kyoto. + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + } + + @Test + void testMorphemesCarryDictionaryFeatures() { + final ListSource strings are written as Unicode escapes to keep this file ASCII-only. The + * Japanese fixture words are Tokyo (U+6771 U+4EAC), Kyoto (U+4EAC U+90FD), east + * (U+6771), the metropolis suffix (U+90FD), the case particle ni (U+306B), and the + * verb iku, to go (U+884C U+304F); the Chinese fixture words are wo, I (U+6211), + * laidao, arrive (U+6765 U+5230), Beijing (U+5317 U+4EAC), and Tiananmen + * (U+5929 U+5B89 U+95E8).
+ */ +public class LatticeUsageExampleTest { + + /** + * Walks the full MeCab-format flow: package a miniature Japanese dictionary as a + * {@code tar.gz} archive, install it from a file URI, load it, and tokenize. The + * segmentation must pick the cheaper path (Tokyo plus the metropolis suffix) over + * the competing reading (east plus Kyoto), the spans must be in original text + * coordinates, and the morphemes must carry the dictionary's feature columns. + */ + @Test + void testInstallLoadAndTokenizeAMecabFormatDictionary(@TempDir Path work) + throws IOException { + // A minimal but complete dictionary: one lexicon file plus the three definition + // files every MeCab-format distribution contains, wrapped like a release archive. + final byte[] archive = TarGzArchives.gzippedTar(new String[][] { + {"mini-dict-0.1/lexicon.csv", String.join("\n", + "\u6771\u4EAC,0,0,3000,noun,proper", + "\u4EAC\u90FD,0,0,3000,noun,proper", + "\u6771,0,0,6000,noun,common", + "\u90FD,0,0,4000,noun,suffix", + "\u306B,0,0,1000,particle,case", + "\u884C\u304F,0,0,3000,verb,base", + "")}, + {"mini-dict-0.1/matrix.def", "1 1\n0 0 0\n"}, + {"mini-dict-0.1/char.def", String.join("\n", + "DEFAULT 0 1 0", + "KANJI 0 0 2", + "HIRAGANA 0 1 0", + "", + "0x3041..0x3096 HIRAGANA", + "0x4E00..0x9FFF KANJI", + "")}, + {"mini-dict-0.1/unk.def", String.join("\n", + "DEFAULT,0,0,10000,symbol,unknown", + "KANJI,0,0,8000,noun,unknown", + "HIRAGANA,0,0,9000,particle,unknown", + "")}, + {"mini-dict-0.1/README", "not a dictionary payload file"}}); + final Path archiveFile = work.resolve("mini-dict-0.1.tar.gz"); + Files.write(archiveFile, archive); + + // Install: fetch the archive from the user-chosen location, verify its digest, + // and unpack the payload. + final Path dictionaryDirectory = work.resolve("dictionary"); + final int extracted = MecabDictionaryInstaller.install( + archiveFile.toUri(), dictionaryDirectory, DigestTestUtil.sha512(archive)); + Assertions.assertEquals(4, extracted); + + // Load and tokenize; both views must agree and stay in original coordinates. + final LatticeTokenizer tokenizer = + new LatticeTokenizer(MecabDictionary.load(dictionaryDirectory)); + final String text = "\u6771\u4EAC\u90FD\u306B\u884C\u304F"; + Assertions.assertArrayEquals( + new String[] {"\u6771\u4EAC", "\u90FD", "\u306B", "\u884C\u304F"}, + tokenizer.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 2), new Span(2, 3), new Span(3, 4), new Span(4, 6)}, + tokenizer.tokenizePos(text)); + + // The analyze view adds the dictionary's feature columns to every morpheme. + final ListSkipped unless {@code -Dopennlp.download.remote=true} is set, matching + * the gate on {@link opennlp.tools.util.DictionaryCatalog}.
+ */ +public class MecabCatalogEndToEndTest { + + @TempDir + private static Path workDir; + + @BeforeAll + static void requireRemoteOptIn() { + Assumptions.assumeTrue(Boolean.getBoolean("opennlp.download.remote"), + "remote downloads are opt-in via -Dopennlp.download.remote=true"); + } + + @Test + void testIpadicInstallsLoadsAndSegments() throws Exception { + MecabDictionary dict = installAndLoad("mecab.ipadic", Charset.forName("EUC-JP")); + LatticeTokenizer tokenizer = new LatticeTokenizer(dict); + + String text = "すもももももももものうち"; + ListSource strings are written as Unicode escapes to keep this file ASCII-only; the + * class works over the same miniature Chinese frequency lexicon as the sibling usage + * example, whose javadoc spells out each fixture word.
+ */ +public class UnigramSegmenterTest { + + private static final String LEXICON = String.join("\n", + "\u6211 5000 r", + "\u6765\u5230 2000 v", + "\u5317\u4EAC 3000 ns", + "\u6E05\u534E\u5927\u5B66 800 nt", + "\u6E05\u534E 400 ns", + "\u534E\u5927 100 ns", + "\u5927\u5B66 1500 n", + "\u7684 9000 uj", + ""); + + private static UnigramSegmenter segmenter; + + @BeforeAll + static void loadLexicon() throws IOException { + segmenter = UnigramSegmenter.load( + new ByteArrayInputStream(LEXICON.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + } + + @Test + void testPrefersWholeWordsOverFragments() { + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); + } + + @Test + void testSpansStayInOriginalCoordinates() { + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 3), new Span(3, 5), new Span(5, 9)}, + segmenter.tokenizePos("\u6211\u6765\u5230\u5317\u4EAC\u6E05\u534E\u5927\u5B66")); + } + + @Test + void testUnknownCharactersFallBackToSingles() { + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u7231", "\u5317\u4EAC"}, + segmenter.tokenize("\u6211\u7231\u5317\u4EAC")); + } + + @Test + void testWhitespaceSeparates() { + Assertions.assertArrayEquals( + new String[] {"\u5317\u4EAC", "\u5927\u5B66"}, + segmenter.tokenize("\u5317\u4EAC \u5927\u5B66")); + Assertions.assertEquals(0, segmenter.tokenizePos(" ").length); + } + + /** + * Verifies that empty input yields empty results from both views of the segmenter. + */ + @Test + void testEmptyInputYieldsEmptyResults() { + Assertions.assertArrayEquals(new String[0], segmenter.tokenize("")); + Assertions.assertArrayEquals(new Span[0], segmenter.tokenizePos("")); + } + + /** + * Verifies single-character input for a listed word and for a character the + * lexicon does not know: both come back as exactly one token covering + * {@code [0, 1)}. + */ + @Test + void testSingleCharacterInput() { + Assertions.assertArrayEquals(new String[] {"\u6211"}, segmenter.tokenize("\u6211")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u6211")); + Assertions.assertArrayEquals(new String[] {"\u7231"}, segmenter.tokenize("\u7231")); + Assertions.assertArrayEquals(new Span[] {new Span(0, 1)}, segmenter.tokenizePos("\u7231")); + } + + /** + * Verifies input made entirely of characters absent from the lexicon: every + * character becomes its own single-character token, since only the unknown + * fallback is available. + */ + @Test + void testEntirelyUnknownInputFallsBackToSingleCharacters() { + Assertions.assertArrayEquals( + new String[] {"x", "y", "z"}, + segmenter.tokenize("xyz")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 3)}, + segmenter.tokenizePos("xyz")); + } + + /** + * Verifies a mixed run of known and unknown text inside one whitespace-free + * stretch: the unknown character becomes a single token while the listed words + * around it, including the longest listed compound, stay intact. + */ + @Test + void testMixedKnownAndUnknownRuns() { + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u7231", "\u6E05\u534E\u5927\u5B66"}, + segmenter.tokenize("\u6211\u7231\u6E05\u534E\u5927\u5B66")); + Assertions.assertArrayEquals(new Span[] { + new Span(0, 1), new Span(1, 2), new Span(2, 6)}, + segmenter.tokenizePos("\u6211\u7231\u6E05\u534E\u5927\u5B66")); + } + + /** + * Verifies that spans keep original text coordinates when the content does not + * start at position zero because of leading whitespace. + */ + @Test + void testSpansStayOriginalAfterLeadingWhitespace() { + final String text = " \u6211\u6765\u5230\u5317\u4EAC"; + Assertions.assertArrayEquals( + new String[] {"\u6211", "\u6765\u5230", "\u5317\u4EAC"}, + segmenter.tokenize(text)); + Assertions.assertArrayEquals(new Span[] { + new Span(2, 3), new Span(3, 5), new Span(5, 7)}, + segmenter.tokenizePos(text)); + } + + @ParameterizedTest(name = "lexicon content \"{0}\"") + @ValueSource(strings = {"word\n", "word abc\n", "word 0\n", "\n\n"}) + void testMalformedLexiconsFailLoud(String lexicon) { + Assertions.assertThrows(IOException.class, () -> UnigramSegmenter.load( + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8)); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((Path) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((Path) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(Path.of("words.txt"), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load((InputStream) null, StandardCharsets.UTF_8)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> UnigramSegmenter.load(new ByteArrayInputStream(new byte[0]), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenize(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> segmenter.tokenizePos(null)); + } + + /** + * Verifies that the unknown-character fallback advances one code point, never one + * code unit: a supplementary character absent from the lexicon comes back as one + * span over both of its surrogate halves, and no span boundary ever lands between + * them. + */ + @Test + void testUnknownSupplementaryCharacterIsNeverSplit() { + // U+20BB7, a CJK extension B ideograph, written as its surrogate pair + final String text = "\uD842\uDFB7\uD842\uDFB7"; + final Span[] spans = segmenter.tokenizePos(text); + for (final Span span : spans) { + Assertions.assertEquals(0, span.getStart() % 2, + "span must start on a code point boundary: " + span); + Assertions.assertEquals(0, span.getEnd() % 2, + "span must end on a code point boundary: " + span); + } + int covered = 0; + for (final Span span : spans) { + covered += span.length(); + } + Assertions.assertEquals(text.length(), covered); + } + + /** + * Pins Unicode-whitespace trimming of lexicon lines: a leading ideographic space + * (U+3000), common in hand-edited CJK text files, is stripped like ASCII whitespace, + * so the entry loads rather than failing as a malformed count. + */ + @Test + void testLeadingIdeographicSpaceIsTrimmed() throws IOException { + // U+3000 ideographic space, then the fixture word U+6211 and its count + final String lexicon = "\u3000\u6211 5000 r\n"; + final UnigramSegmenter loaded = UnigramSegmenter.load( + new ByteArrayInputStream(lexicon.getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8); + Assertions.assertArrayEquals(new String[] {"\u6211"}, loaded.tokenize("\u6211")); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java new file mode 100644 index 0000000000..d721873951 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests the opt-in dictionary catalog against an in-memory properties file and a + * local file URI so no network access is required. + */ +public class DictionaryCatalogTest { + + /** + * Verifies that a catalog download without the remote-download property fails with + * the property name in the message. + * + * @param dir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture catalog cannot be prepared. + */ + @Test + void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception { + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + final DictionaryCatalog loaded = demoCatalog(dir, payload); + + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + try { + final IOException e = Assertions.assertThrows(IOException.class, + () -> loaded.download("demo", dir.resolve("out.bin"))); + Assertions.assertTrue(e.getMessage().contains(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY)); + } finally { + restore(previous); + } + } + + /** + * Verifies that an enabled catalog download fetches the entry and writes the + * digest-verified bytes to the target. + * + * @param dir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture catalog cannot be prepared or fetched. + */ + @Test + void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception { + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + final DictionaryCatalog loaded = demoCatalog(dir, payload); + final Path target = dir.resolve("out.bin"); + + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, "true"); + try { + loaded.download("demo", target); + Assertions.assertArrayEquals(payload, Files.readAllBytes(target)); + } finally { + restore(previous); + } + } + + /** + * Verifies that the shipped catalog holds the MeCab and Hunspell entries, each with + * a full-length SHA-512 digest. + * + * @throws IOException Thrown if the shipped catalog fails to load. + */ + @Test + void testDefaultCatalogContainsMecabAndHunspellEntries() throws IOException { + final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); + Assertions.assertTrue(catalog.ids().contains("mecab.ipadic")); + Assertions.assertTrue(catalog.ids().contains("mecab.ko-dic")); + Assertions.assertTrue(catalog.ids().contains("hunspell.en_US.aff")); + Assertions.assertEquals(128, catalog.get("mecab.ipadic").sha512().length()); + Assertions.assertEquals(128, catalog.get("hunspell.en_US.dic").sha512().length()); + } + + /** + * Builds a one-entry catalog whose URL is a local file holding {@code payload}, so + * downloads need no network. + * + * @param dir The directory to write the payload file into. + * @param payload The bytes the catalog entry points at. + * @return The loaded catalog. Never {@code null}. + * @throws IOException Thrown if the payload file cannot be written. + */ + private static DictionaryCatalog demoCatalog(Path dir, byte[] payload) + throws IOException { + final Path source = dir.resolve("dict.bin"); + Files.write(source, payload); + final String catalog = "demo.url=" + source.toUri() + "\n" + + "demo.sha512=" + DigestTestUtil.sha512(payload) + "\n"; + return DictionaryCatalog.load( + new ByteArrayInputStream(catalog.getBytes(StandardCharsets.UTF_8))); + } + + /** + * Restores the remote-download property to its value before the test. + * + * @param previous The saved value, or {@code null} when the property was unset. + */ + private static void restore(String previous) { + if (previous == null) { + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + } else { + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, previous); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java new file mode 100644 index 0000000000..af85b09a8e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** + * Computes SHA-512 digests for test fixtures. + */ +public final class DigestTestUtil { + + private DigestTestUtil() { + } + + /** + * {@return the SHA-512 digest of {@code bytes} as 128 lowercase hex digits} + * + * @param bytes The content to digest. Must not be {@code null}. + */ + public static String sha512(byte[] bytes) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-512").digest(bytes)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java new file mode 100644 index 0000000000..5451cabc5f --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Pins {@link DownloadUtil#download(java.net.URI, Path, String)} against local file URIs + * so digest verification and the size ceiling are covered without a network. + */ +public class DownloadUtilFileTest { + + /** The fixture bytes the download tests serve and digest. */ + private static final byte[] PAYLOAD = "dictionary-bytes".getBytes(StandardCharsets.UTF_8); + + /** + * Verifies that a download whose bytes match the expected digest lands in the + * target file. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or fetched. + */ + @Test + void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + DownloadUtil.download(source.toUri(), target, DigestTestUtil.sha512(PAYLOAD)); + + Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); + } + + /** + * Verifies that a digest mismatch fails the download and leaves no target file + * behind. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ + @Test + void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + final String wrong = DigestTestUtil.sha512("other".getBytes(StandardCharsets.UTF_8)); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> DownloadUtil.download(source.toUri(), target, wrong)); + Assertions.assertTrue(e.getMessage().contains("SHA512 checksum validation failed")); + Assertions.assertTrue(Files.notExists(target)); + } + + /** Verifies that a {@code null} digest is rejected with the documented exception. */ + @Test + void testDownloadRequiresSha512() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> DownloadUtil.download(Path.of("x").toUri(), Path.of("y"), null)); + } + + /** + * Verifies that a digest shorter than 128 hex digits is rejected before anything is + * fetched. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ + @Test + void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> DownloadUtil.download(source.toUri(), dir.resolve("target.bin"), "abc123")); + } + + /** + * Verifies that a source larger than the byte ceiling fails the download and leaves + * no target file behind. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ + @Test + void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> DownloadUtil.download(source.toUri(), target, + DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length - 1)); + Assertions.assertTrue(e.getMessage().contains("exceeds safe limit")); + Assertions.assertTrue(Files.notExists(target)); + } + + /** + * Pins the inclusive byte ceiling: a source of exactly the ceiling's size still + * downloads. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or fetched. + */ + @Test + void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + DownloadUtil.download(source.toUri(), target, + DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length); + + Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); + } + + /** Verifies that a positive property value overrides the fallback limit. */ + @Test + void testConfiguredLimitOverridesFromProperty() { + final String property = "opennlp.test.limit.override"; + System.setProperty(property, "1024"); + try { + Assertions.assertEquals(1024L, DownloadUtil.configuredLimit(property, 7L)); + } finally { + System.clearProperty(property); + } + } + + /** Verifies that an unset property falls back to the given default. */ + @Test + void testConfiguredLimitFallsBackWhenAbsent() { + Assertions.assertEquals(7L, + DownloadUtil.configuredLimit("opennlp.test.limit.absent", 7L)); + } + + /** Verifies that blank, non-numeric, and non-positive values fall back. */ + @ParameterizedTest(name = "value \"{0}\" falls back") + @ValueSource(strings = {"", " ", "abc", "-1", "0"}) + void testConfiguredLimitRejectsInvalidValues(String invalid) { + final String property = "opennlp.test.limit.invalid"; + System.setProperty(property, invalid); + try { + Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L)); + } finally { + System.clearProperty(property); + } + } + + /** Pins the default download ceiling of 512 MiB when no override property is set. */ + @Test + void testDefaultBudgetsWithoutOverrides() { + Assertions.assertEquals(512L * 1024 * 1024, DownloadUtil.MAX_DOWNLOAD_BYTES); + } +} diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index cd1d8a2ddf..7068f31643 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -538,5 +538,46 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { +LatticeTokenizer scores paths over a MeCab-format dictionary and
+ emits the cheapest segmentation with spans in original text coordinates.
+ UnigramSegmenter does the same from a plain frequency lexicon.
+ Install a dictionary archive with MecabDictionaryInstaller, load it
+ as a MecabDictionary, and tokenize. Remote archives require an
+ expected SHA-512 digest; the opt-in catalog
+ (installFromCatalog) also needs
+ -Dopennlp.download.remote=true.
+ matrix.def must list a cost for every declared cell; matrix
+ dimensions plus lexicon size are bounded by the shared
+ ResourceLimits.MAX_ENTRIES limit and the matrix cell count
+ by ResourceLimits.MAX_MATRIX_CELLS.
+ Extraction caps per-entry size, total bytes, entry count, and gzip expansion;
+ the byte ceilings default to 512 MiB per download and entry and 2 GiB total,
+ and can be raised at JVM startup via
+ opennlp.download.max.bytes,
+ opennlp.install.max.entry.bytes, and
+ opennlp.install.max.total.bytes for larger dictionaries such as
+ UniDic.
+ Load also rejects an unk.def template for a category
+ char.def never defined, and accepts MeCab-quoted CSV fields.
+ LatticeUsageExampleTest asserts the install-load-tokenize and
+ lexicon flows shown here.
+