From 3dc0950e7ce067b30b71a3709ef3e2780c6c9155 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:05:37 -0400 Subject: [PATCH 01/24] OPENNLP-1893: Hunspell-format affix engine over user-supplied dictionaries A clean-room reader for .dic and .aff files with PFX/SFX rules, strip strings, character-class conditions matched by a single scan, cross products, and char, long, and num flag modes. No dictionary data is bundled: users point at their own files, so dictionary licenses never attach to the jar. Unsupported affix features fail closed, missing analyses rather than inventing them. (cherry picked from commit 0ecc39c7febdeaaafe1cfbdf052c9dd06ffbe4d4) --- .../stemmer/hunspell/AffixCondition.java | 130 ++++++ .../stemmer/hunspell/HunspellDictionary.java | 430 ++++++++++++++++++ .../stemmer/hunspell/HunspellStemmer.java | 165 +++++++ .../hunspell/HunspellStemmerFactory.java | 52 +++ .../stemmer/hunspell/HunspellStemmerTest.java | 181 ++++++++ 5 files changed, 958 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java new file mode 100644 index 0000000000..8da9a34968 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * One parsed affix condition: a fixed-length sequence of literal characters and + * bracketed character classes, matched with a single scan and no regular expressions. + * A suffix condition anchors at the end of the candidate stem, a prefix condition at + * its start; the condition {@code .} matches everything. + */ +final class AffixCondition { + + private static final AffixCondition ANY = new AffixCondition(new char[0][], null, true); + + /** Per position: the accepted characters, or {@code null} for any character. */ + private final char[][] accepted; + /** Per position with a class: whether the class is negated; {@code null} rows unused. */ + private final boolean[] negated; + private final boolean suffix; + + private AffixCondition(char[][] accepted, boolean[] negated, boolean suffix) { + this.accepted = accepted; + this.negated = negated; + this.suffix = suffix; + } + + /** + * Parses a condition field. + * + * @param pattern The condition text from the affix rule. + * @param suffix Whether the owning rule is a suffix rule. + * @param lineNumber The affix file line, for error messages. + * @return The parsed condition. Never {@code null}. + * @throws IOException Thrown if a character class is unterminated. + */ + static AffixCondition parse(String pattern, boolean suffix, int lineNumber) + throws IOException { + if (".".equals(pattern)) { + return ANY; + } + final List positions = new ArrayList<>(); + final List negations = new ArrayList<>(); + int i = 0; + while (i < pattern.length()) { + final char c = pattern.charAt(i); + if (c == '[') { + final int end = pattern.indexOf(']', i + 1); + if (end < 0) { + throw new IOException("unterminated character class at line " + lineNumber); + } + String members = pattern.substring(i + 1, end); + boolean negate = false; + if (members.startsWith("^")) { + negate = true; + members = members.substring(1); + } + positions.add(members.toCharArray()); + negations.add(negate); + i = end + 1; + } else if (c == '.') { + positions.add(null); + negations.add(false); + i++; + } else { + positions.add(new char[] {c}); + negations.add(false); + i++; + } + } + final char[][] accepted = positions.toArray(new char[0][]); + final boolean[] negated = new boolean[accepted.length]; + for (int p = 0; p < negated.length; p++) { + negated[p] = negations.get(p); + } + return new AffixCondition(accepted, negated, suffix); + } + + /** + * Tests a candidate stem against the condition at its anchored side. + * + * @param stem The candidate stem after affix removal and strip restoration. + * @return {@code true} if the stem satisfies the condition. + */ + boolean matches(String stem) { + if (accepted.length == 0) { + return true; + } + if (stem.length() < accepted.length) { + return false; + } + final int offset = suffix ? stem.length() - accepted.length : 0; + for (int p = 0; p < accepted.length; p++) { + final char[] members = accepted[p]; + if (members == null) { + continue; + } + final char c = stem.charAt(offset + p); + boolean member = false; + for (final char candidate : members) { + if (candidate == c) { + member = true; + break; + } + } + if (member == negated[p]) { + return false; + } + } + return true; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java new file mode 100644 index 0000000000..8ac526fc38 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.util.StringUtil; + +/** + * An immutable, in-memory Hunspell-format dictionary: the word list of a {@code .dic} + * file and the prefix and suffix rules of its {@code .aff} companion, loaded from + * user-supplied files. The engine is a clean-room implementation of the documented + * format; no dictionary data is bundled, so the dictionaries' own licenses never attach + * to this library. + * + *

Supported affix features: {@code PFX} and {@code SFX} rules with strip strings, + * character-class conditions, and cross-product combination of one prefix with one + * suffix; {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; + * the {@code SET} encoding declaration. Compounding, continuation classes, and + * conversion tables are not interpreted in this version; rules using them simply do + * not fire, so unsupported analyses are missed rather than invented.

+ * + *

Instances are immutable and safe to share between threads.

+ * + * @see HunspellStemmer + * @see HunspellStemmerFactory + * @since 3.0.0 + */ +public final class HunspellDictionary { + + /** One parsed affix rule; {@code affix} is the surface material added to the stem. */ + record Affix(int flag, boolean crossProduct, String strip, String affix, + AffixCondition condition) { + } + + private final Map> entries; + private final List prefixes; + private final List suffixes; + + private HunspellDictionary(Map> entries, List prefixes, + List suffixes) { + this.entries = entries; + this.prefixes = prefixes; + this.suffixes = suffixes; + } + + /** + * Loads a dictionary from its two files. + * + * @param affixFile The {@code .aff} affix file. Must not be {@code null}. + * @param dictionaryFile The {@code .dic} word list. Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails or a file is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static HunspellDictionary load(Path affixFile, Path dictionaryFile) + throws IOException { + if (affixFile == null || dictionaryFile == null) { + throw new IllegalArgumentException("affixFile and dictionaryFile must not be null"); + } + try (InputStream affix = Files.newInputStream(affixFile); + InputStream dictionary = Files.newInputStream(dictionaryFile)) { + return load(affix, dictionary); + } + } + + /** + * Loads a dictionary from its two streams. + * + * @param affixStream The {@code .aff} affix content. Must not be {@code null}. Not + * closed. + * @param dictionaryStream The {@code .dic} word list content. Must not be + * {@code null}. Not closed. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if reading fails or the content is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static HunspellDictionary load(InputStream affixStream, + InputStream dictionaryStream) throws IOException { + if (affixStream == null || dictionaryStream == null) { + throw new IllegalArgumentException("streams must not be null"); + } + final byte[] affixBytes = readAll(affixStream); + final Charset charset = declaredCharset(affixBytes); + final AffixFile affix = parseAffix(new String(affixBytes, charset)); + final Map> entries = + parseWordList(new String(readAll(dictionaryStream), charset), affix.flagMode); + return new HunspellDictionary(entries, List.copyOf(affix.prefixes), + List.copyOf(affix.suffixes)); + } + + /** + * Looks up a word's flag sets. + * + * @param word The word exactly as listed. + * @return The flag sets of all matching entries, or {@code null} when absent. + */ + List lookup(String word) { + return entries.get(word); + } + + /** @return The prefix rules. */ + List prefixes() { + return prefixes; + } + + /** @return The suffix rules. */ + List suffixes() { + return suffixes; + } + + /** + * Checks whether any of a word's flag sets carries a flag. + * + * @param flagSets The flag sets from {@link #lookup(String)}. + * @param flag The flag to look for. + * @return {@code true} if some flag set contains the flag. + */ + static boolean hasFlag(List flagSets, int flag) { + for (final int[] flags : flagSets) { + for (final int candidate : flags) { + if (candidate == flag) { + return true; + } + } + } + return false; + } + + private static byte[] readAll(InputStream in) throws IOException { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + final byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) >= 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + /** Finds the {@code SET} declaration by scanning the raw bytes as ASCII. */ + private static Charset declaredCharset(byte[] affixBytes) throws IOException { + final String ascii = new String(affixBytes, StandardCharsets.US_ASCII); + for (final String line : splitLines(ascii)) { + final String trimmed = line.trim(); + if (trimmed.startsWith("SET ") || trimmed.startsWith("SET\t")) { + final String name = trimmed.substring(4).trim(); + try { + return Charset.forName(name); + } catch (RuntimeException e) { + throw new IOException("unsupported SET encoding: " + name, e); + } + } + } + return StandardCharsets.UTF_8; + } + + /** The flag encodings a dictionary may declare. */ + private enum FlagMode { + CHAR, LONG, NUM + } + + /** The parsed affix file content. */ + private static final class AffixFile { + private final List prefixes = new ArrayList<>(); + private final List suffixes = new ArrayList<>(); + private FlagMode flagMode = FlagMode.CHAR; + } + + private static AffixFile parseAffix(String content) throws IOException { + final AffixFile result = new AffixFile(); + final String[] lines = splitLines(content); + int i = 0; + while (i < lines.length) { + final String[] fields = split(lines[i]); + if (fields.length == 0 || fields[0].startsWith("#")) { + i++; + continue; + } + switch (fields[0]) { + case "FLAG": + if (fields.length < 2) { + throw new IOException("FLAG line without a mode at line " + (i + 1)); + } + result.flagMode = switch (fields[1]) { + case "long" -> FlagMode.LONG; + case "num" -> FlagMode.NUM; + default -> throw new IOException( + "unsupported FLAG mode '" + fields[1] + "' at line " + (i + 1)); + }; + i++; + break; + case "PFX": + case "SFX": + i = parseAffixBlock(lines, i, fields, result); + break; + default: + i++; + break; + } + } + return result; + } + + /** Parses one PFX or SFX header and its rule lines; returns the next line index. */ + private static int parseAffixBlock(String[] lines, int index, String[] header, + AffixFile result) throws IOException { + if (header.length < 4) { + throw new IOException("malformed affix header at line " + (index + 1)); + } + final boolean suffix = "SFX".equals(header[0]); + final int flag = parseFlag(header[1], result.flagMode, index + 1); + final boolean crossProduct = "Y".equals(header[2]); + final int count; + try { + count = Integer.parseInt(header[3]); + } catch (NumberFormatException e) { + throw new IOException("malformed affix rule count at line " + (index + 1), e); + } + int line = index + 1; + for (int rule = 0; rule < count; rule++, line++) { + if (line >= lines.length) { + throw new IOException("affix block truncated at line " + (line + 1)); + } + final String[] fields = split(lines[line]); + if (fields.length < 5 || !fields[0].equals(header[0])) { + throw new IOException("malformed affix rule at line " + (line + 1)); + } + final String strip = "0".equals(fields[2]) ? "" : fields[2]; + String affixText = fields[3]; + final int continuation = affixText.indexOf('/'); + if (continuation >= 0) { + affixText = affixText.substring(0, continuation); + } + if ("0".equals(affixText)) { + affixText = ""; + } + final Affix affix = new Affix(flag, crossProduct, strip, affixText, + AffixCondition.parse(fields[4], suffix, line + 1)); + if (suffix) { + result.suffixes.add(affix); + } else { + result.prefixes.add(affix); + } + } + return line; + } + + private static Map> parseWordList(String content, + FlagMode flagMode) throws IOException { + final String[] lines = splitLines(content); + final Map> entries = new HashMap<>(); + int start = 0; + if (lines.length > 0 && isCount(lines[0].trim())) { + start = 1; + } + for (int i = start; i < lines.length; i++) { + final String line = lines[i].trim(); + if (line.isEmpty()) { + continue; + } + String word = line; + int[] flags = new int[0]; + final int slash = unescapedSlash(line); + if (slash >= 0) { + word = line.substring(0, slash); + String flagText = line.substring(slash + 1); + final int fieldEnd = whitespaceIndex(flagText); + if (fieldEnd >= 0) { + flagText = flagText.substring(0, fieldEnd); + } + flags = parseFlags(flagText, flagMode, i + 1); + } else { + final int fieldEnd = whitespaceIndex(word); + if (fieldEnd >= 0) { + word = word.substring(0, fieldEnd); + } + } + entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) + .add(flags); + } + return entries; + } + + private static boolean isCount(String line) { + if (line.isEmpty()) { + return false; + } + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) < '0' || line.charAt(i) > '9') { + return false; + } + } + return true; + } + + /** Finds the first {@code /} that is not escaped as {@code \/}. */ + private static int unescapedSlash(String line) { + for (int i = 0; i < line.length(); i++) { + if (line.charAt(i) == '/' && (i == 0 || line.charAt(i - 1) != '\\')) { + return i; + } + } + return -1; + } + + private static int whitespaceIndex(String text) { + for (int i = 0; i < text.length(); i++) { + if (StringUtil.isWhitespace(text.charAt(i))) { + return i; + } + } + return -1; + } + + private static int[] parseFlags(String text, FlagMode mode, int lineNumber) + throws IOException { + switch (mode) { + case NUM: { + final String[] parts = splitOn(text, ','); + final int[] flags = new int[parts.length]; + for (int i = 0; i < parts.length; i++) { + try { + flags[i] = Integer.parseInt(parts[i].trim()); + } catch (NumberFormatException e) { + throw new IOException("malformed numeric flag at line " + lineNumber, e); + } + } + return flags; + } + case LONG: { + if (text.length() % 2 != 0) { + throw new IOException("odd long-flag run at line " + lineNumber); + } + final int[] flags = new int[text.length() / 2]; + for (int i = 0; i < flags.length; i++) { + flags[i] = (text.charAt(2 * i) << 16) | text.charAt(2 * i + 1); + } + return flags; + } + default: { + final int[] flags = new int[text.length()]; + for (int i = 0; i < flags.length; i++) { + flags[i] = text.charAt(i); + } + return flags; + } + } + } + + private static int parseFlag(String text, FlagMode mode, int lineNumber) + throws IOException { + final int[] flags = parseFlags(text, mode, lineNumber); + if (flags.length != 1) { + throw new IOException("expected exactly one flag at line " + lineNumber); + } + return flags[0]; + } + + /** Splits text into lines with a single character scan, tolerating CRLF endings. */ + private static String[] splitLines(String content) { + final List lines = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= content.length(); i++) { + if (i == content.length() || content.charAt(i) == '\n') { + int end = i; + if (end > start && content.charAt(end - 1) == '\r') { + end--; + } + lines.add(content.substring(start, end)); + start = i + 1; + } + } + return lines.toArray(new String[0]); + } + + /** Splits text on a separator character with a single character scan. */ + private static String[] splitOn(String text, char separator) { + final List parts = new ArrayList<>(); + int start = 0; + for (int i = 0; i <= text.length(); i++) { + if (i == text.length() || text.charAt(i) == separator) { + parts.add(text.substring(start, i)); + start = i + 1; + } + } + return parts.toArray(new String[0]); + } + + /** Splits a line on whitespace with a single character scan. */ + private static String[] split(String line) { + final List parts = new ArrayList<>(); + int start = -1; + for (int i = 0; i <= line.length(); i++) { + if (i == line.length() || StringUtil.isWhitespace(line.charAt(i))) { + if (start >= 0) { + parts.add(line.substring(start, i)); + start = -1; + } + } else if (start < 0) { + start = i; + } + } + return parts.toArray(new String[0]); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java new file mode 100644 index 0000000000..22d32950b7 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.hunspell.HunspellDictionary.Affix; +import opennlp.tools.util.StringUtil; + +/** + * A dictionary-backed {@link Stemmer} over a {@link HunspellDictionary}: a surface form + * is reduced to the dictionary words it can be derived from by removing one suffix, one + * prefix, or a cross-product combination of both. + * + *

{@link #stem(CharSequence)} returns the first analysis, preferring the word's own + * dictionary entry; {@link #stemAll(CharSequence)} returns every distinct analysis. A + * word with no analysis is returned unchanged, so the stemmer degrades to identity on + * unknown vocabulary. Capitalized forms also try their lowercase variant.

+ * + *

The stemmer reads only immutable dictionary state and is safe to share between + * threads, satisfying the single-thread confinement contract trivially.

+ * + * @since 3.0.0 + */ +public class HunspellStemmer implements Stemmer { + + private final HunspellDictionary dictionary; + + /** + * Initializes the stemmer. + * + * @param dictionary The dictionary to analyze against. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public HunspellStemmer(HunspellDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + @Override + public CharSequence stem(CharSequence word) { + final List analyses = stemAll(word); + return analyses.get(0); + } + + @Override + public List stemAll(CharSequence word) { + if (word == null) { + throw new IllegalArgumentException("word must not be null"); + } + final String surface = word.toString(); + final Set analyses = new LinkedHashSet<>(); + for (final String variant : variants(surface)) { + analyze(variant, analyses); + } + if (analyses.isEmpty()) { + return List.of(surface); + } + return List.copyOf(new ArrayList(analyses)); + } + + /** Collects the case variants to analyze: the surface form, then its lowercase. */ + private static List variants(String surface) { + final String lowered = StringUtil.toLowerCase(surface); + return lowered.equals(surface) ? List.of(surface) : List.of(surface, lowered); + } + + /** Runs direct lookup, suffix removal, prefix removal, and cross products. */ + private void analyze(String word, Set analyses) { + if (dictionary.lookup(word) != null) { + analyses.add(word); + } + for (final Affix suffix : dictionary.suffixes()) { + final String stem = removeSuffix(word, suffix); + if (stem == null) { + continue; + } + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { + analyses.add(stem); + } + } + for (final Affix prefix : dictionary.prefixes()) { + final String stem = removePrefix(word, prefix); + if (stem == null) { + continue; + } + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, prefix.flag())) { + analyses.add(stem); + } + if (!prefix.crossProduct()) { + continue; + } + for (final Affix suffix : dictionary.suffixes()) { + if (!suffix.crossProduct()) { + continue; + } + final String doubleStem = removeSuffix(stem, suffix); + if (doubleStem == null) { + continue; + } + final List both = dictionary.lookup(doubleStem); + if (both != null && HunspellDictionary.hasFlag(both, prefix.flag()) + && HunspellDictionary.hasFlag(both, suffix.flag())) { + analyses.add(doubleStem); + } + } + } + } + + /** + * Undoes one suffix rule. + * + * @param word The surface form. + * @param suffix The rule to undo. + * @return The candidate stem, or {@code null} when the rule does not apply. + */ + private static String removeSuffix(String word, Affix suffix) { + if (suffix.affix().isEmpty() || !word.endsWith(suffix.affix()) + || word.length() - suffix.affix().length() + suffix.strip().length() == 0) { + return null; + } + final String stem = + word.substring(0, word.length() - suffix.affix().length()) + suffix.strip(); + return suffix.condition().matches(stem) ? stem : null; + } + + /** + * Undoes one prefix rule. + * + * @param word The surface form. + * @param prefix The rule to undo. + * @return The candidate stem, or {@code null} when the rule does not apply. + */ + private static String removePrefix(String word, Affix prefix) { + if (prefix.affix().isEmpty() || !word.startsWith(prefix.affix()) + || word.length() - prefix.affix().length() + prefix.strip().length() == 0) { + return null; + } + final String stem = prefix.strip() + word.substring(prefix.affix().length()); + return prefix.condition().matches(stem) ? stem : null; + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java new file mode 100644 index 0000000000..eb0a8d596e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.StemmerFactory; + +/** + * The shareable handle for Hunspell stemming: holds one immutable + * {@link HunspellDictionary} and hands out {@link HunspellStemmer} instances over it. + * + *

The factory is immutable and safe to share across threads.

+ * + * @since 3.0.0 + */ +public class HunspellStemmerFactory implements StemmerFactory { + + private final HunspellDictionary dictionary; + + /** + * Initializes the factory. + * + * @param dictionary The dictionary to stem against. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code dictionary} is {@code null}. + */ + public HunspellStemmerFactory(HunspellDictionary dictionary) { + if (dictionary == null) { + throw new IllegalArgumentException("dictionary must not be null"); + } + this.dictionary = dictionary; + } + + @Override + public Stemmer newStemmer() { + return new HunspellStemmer(dictionary); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java new file mode 100644 index 0000000000..acd2561171 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import opennlp.tools.stemmer.Stemmer; + +/** + * Tests the affix engine against a project-authored miniature dictionary; no external + * dictionary data is involved. + */ +public class HunspellStemmerTest { + + private static final String AFFIX = String.join("\n", + "# project-authored test fixture", + "SET UTF-8", + "", + "PFX U Y 1", + "PFX U 0 un .", + "", + "SFX S Y 3", + "SFX S 0 s [^sxy]", + "SFX S y ies y", + "SFX S 0 es [sx]", + "", + "SFX G Y 2", + "SFX G 0 ing [^e]", + "SFX G e ing e", + ""); + + private static final String WORDS = String.join("\n", + "6", + "lock/USG", + "pony/S", + "make/G", + "cat/S", + "box/S", + "fish", + ""); + + private static HunspellStemmer stemmer; + + @BeforeAll + static void loadDictionary() throws IOException { + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(AFFIX.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(WORDS.getBytes(StandardCharsets.UTF_8))); + stemmer = new HunspellStemmer(dictionary); + } + + @Test + void testSuffixRules() { + Assertions.assertEquals("cat", stemmer.stem("cats").toString()); + Assertions.assertEquals("pony", stemmer.stem("ponies").toString()); + Assertions.assertEquals("box", stemmer.stem("boxes").toString()); + Assertions.assertEquals("make", stemmer.stem("making").toString()); + Assertions.assertEquals("lock", stemmer.stem("locking").toString()); + } + + @Test + void testPrefixAndCrossProduct() { + Assertions.assertEquals("lock", stemmer.stem("unlock").toString()); + Assertions.assertEquals("lock", stemmer.stem("unlocks").toString()); + Assertions.assertEquals("lock", stemmer.stem("unlocking").toString()); + } + + @Test + void testConditionsBlockWrongAnalyses() { + // the s rule requires a stem not ending in s, x, or y + Assertions.assertEquals("boxs", stemmer.stem("boxs").toString()); + // cat carries no G flag, so no ing analysis exists + Assertions.assertEquals("cating", stemmer.stem("cating").toString()); + // fish carries no flags at all + Assertions.assertEquals("fishs", stemmer.stem("fishs").toString()); + } + + @Test + void testDirectLookupAndCase() { + Assertions.assertEquals("fish", stemmer.stem("fish").toString()); + Assertions.assertEquals("cat", stemmer.stem("Cats").toString()); + Assertions.assertEquals("lock", stemmer.stem("Unlocks").toString()); + } + + @Test + void testUnknownWordsPassThroughUnchanged() { + Assertions.assertEquals("zebras", stemmer.stem("zebras").toString()); + Assertions.assertEquals(1, stemmer.stemAll("zebras").size()); + } + + @Test + void testStemAllReportsEveryAnalysis() { + Assertions.assertEquals(1, stemmer.stemAll("unlocks").size()); + Assertions.assertEquals("lock", stemmer.stemAll("unlocks").get(0).toString()); + // the surface form itself is an entry AND an analysis target + Assertions.assertEquals("lock", stemmer.stemAll("lock").get(0).toString()); + } + + @Test + void testNumericFlagMode() throws IOException { + final String affix = String.join("\n", + "SET UTF-8", + "FLAG num", + "SFX 100 Y 1", + "SFX 100 0 s .", + ""); + final String words = String.join("\n", "1", "walk/100,7", ""); + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + Assertions.assertEquals("walk", + new HunspellStemmer(dictionary).stem("walks").toString()); + } + + @Test + void testLongFlagMode() throws IOException { + final String affix = String.join("\n", + "SET UTF-8", + "FLAG long", + "SFX Aa Y 1", + "SFX Aa 0 s .", + ""); + final String words = String.join("\n", "1", "walk/AaBb", ""); + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + Assertions.assertEquals("walk", + new HunspellStemmer(dictionary).stem("walks").toString()); + } + + @Test + void testFactoryHandsOutWorkingStemmers() throws IOException { + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(AFFIX.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(WORDS.getBytes(StandardCharsets.UTF_8))); + final Stemmer fresh = new HunspellStemmerFactory(dictionary).newStemmer(); + Assertions.assertEquals("pony", fresh.stem("ponies").toString()); + } + + @Test + void testMalformedInputFailsLoud() { + Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( + new ByteArrayInputStream("SFX S Y 2\nSFX S 0 s .\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream("1\ncat/S\n".getBytes(StandardCharsets.UTF_8)))); + Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( + new ByteArrayInputStream("SET NO-SUCH-ENCODING\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream("0\n".getBytes(StandardCharsets.UTF_8)))); + Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( + new ByteArrayInputStream("SFX S 0 s [a\n".getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream("0\n".getBytes(StandardCharsets.UTF_8)))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load((java.io.InputStream) null, + (java.io.InputStream) null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HunspellStemmer(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HunspellStemmerFactory(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> stemmer.stemAll(null)); + } +} From ed799928806a79f1a9d2713e31ed51a0bd90cd05 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 15 Jul 2026 13:53:05 -0400 Subject: [PATCH 02/24] OPENNLP-1893: Twofold suffix analysis through Hunspell continuation classes Suffix rules now carry the continuation flags declared on their affix text, and analysis undoes a stacked pair when the inner rule's classes allow the outer one, so derived-then-inflected forms reduce to their dictionary word. (cherry picked from commit b543dea24c5241d5f3431b34be987f6e83fe887e) --- .../stemmer/hunspell/HunspellDictionary.java | 39 ++++++++++++++----- .../stemmer/hunspell/HunspellStemmer.java | 13 +++++++ .../stemmer/hunspell/HunspellStemmerTest.java | 21 ++++++++++ 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 8ac526fc38..02dff67559 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -40,10 +40,11 @@ * *

Supported affix features: {@code PFX} and {@code SFX} rules with strip strings, * character-class conditions, and cross-product combination of one prefix with one - * suffix; {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; - * the {@code SET} encoding declaration. Compounding, continuation classes, and - * conversion tables are not interpreted in this version; rules using them simply do - * not fire, so unsupported analyses are missed rather than invented.

+ * suffix; twofold suffixes through the continuation classes on suffix rules; + * {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; the + * {@code SET} encoding declaration. Compounding and conversion tables are not + * interpreted in this version; rules using them simply do not fire, so unsupported + * analyses are missed rather than invented.

* *

Instances are immutable and safe to share between threads.

* @@ -53,9 +54,25 @@ */ public final class HunspellDictionary { - /** One parsed affix rule; {@code affix} is the surface material added to the stem. */ + /** One parsed affix rule; {@code affix} is the surface material added to the stem, + * and {@code continuation} lists the flags of affixes that may stack on top. */ record Affix(int flag, boolean crossProduct, String strip, String affix, - AffixCondition condition) { + AffixCondition condition, int[] continuation) { + + /** + * Checks whether a further affix may stack on this one. + * + * @param otherFlag The stacking affix's flag. + * @return {@code true} if this affix's continuation classes allow it. + */ + boolean allowsContinuation(int otherFlag) { + for (final int candidate : continuation) { + if (candidate == otherFlag) { + return true; + } + } + return false; + } } private final Map> entries; @@ -252,15 +269,17 @@ private static int parseAffixBlock(String[] lines, int index, String[] header, } final String strip = "0".equals(fields[2]) ? "" : fields[2]; String affixText = fields[3]; - final int continuation = affixText.indexOf('/'); - if (continuation >= 0) { - affixText = affixText.substring(0, continuation); + int[] continuation = new int[0]; + final int slash = affixText.indexOf('/'); + if (slash >= 0) { + continuation = parseFlags(affixText.substring(slash + 1), result.flagMode, line + 1); + affixText = affixText.substring(0, slash); } if ("0".equals(affixText)) { affixText = ""; } final Affix affix = new Affix(flag, crossProduct, strip, affixText, - AffixCondition.parse(fields[4], suffix, line + 1)); + AffixCondition.parse(fields[4], suffix, line + 1), continuation); if (suffix) { result.suffixes.add(affix); } else { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 22d32950b7..ba2dd14eb6 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -100,6 +100,19 @@ private void analyze(String word, Set analyses) { if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { analyses.add(stem); } + for (final Affix inner : dictionary.suffixes()) { + if (!inner.allowsContinuation(suffix.flag())) { + continue; + } + final String doubleStem = removeSuffix(stem, inner); + if (doubleStem == null) { + continue; + } + final List innerFlags = dictionary.lookup(doubleStem); + if (innerFlags != null && HunspellDictionary.hasFlag(innerFlags, inner.flag())) { + analyses.add(doubleStem); + } + } } for (final Affix prefix : dictionary.prefixes()) { final String stem = removePrefix(word, prefix); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index acd2561171..2045f6e645 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -117,6 +117,27 @@ void testStemAllReportsEveryAnalysis() { Assertions.assertEquals("lock", stemmer.stemAll("lock").get(0).toString()); } + @Test + void testTwofoldSuffixesThroughContinuationClasses() throws IOException { + final String affix = String.join("\n", + "SET UTF-8", + "SFX A Y 1", + "SFX A 0 er/B .", + "SFX B Y 1", + "SFX B 0 s .", + ""); + final String words = String.join("\n", "1", "kind/A", ""); + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + final HunspellStemmer twofold = new HunspellStemmer(dictionary); + + Assertions.assertEquals("kind", twofold.stem("kinder").toString()); + Assertions.assertEquals("kind", twofold.stem("kinders").toString()); + // B alone never applies: no entry carries it directly + Assertions.assertEquals("kinds", twofold.stem("kinds").toString()); + } + @Test void testNumericFlagMode() throws IOException { final String affix = String.join("\n", From d37af0d3283433c65186e624ff9c10cc87201f75 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:43:34 -0400 Subject: [PATCH 03/24] OPENNLP-1893: Usage, threading, and malformed-input tests for the Hunspell engine, precise javadoc --- .../stemmer/hunspell/AffixCondition.java | 9 +- .../stemmer/hunspell/HunspellDictionary.java | 112 +++++++++- .../stemmer/hunspell/HunspellStemmer.java | 38 +++- .../hunspell/HunspellStemmerFactory.java | 4 + .../hunspell/HunspellStemmerFactoryTest.java | 179 ++++++++++++++++ .../stemmer/hunspell/HunspellStemmerTest.java | 196 ++++++++++++++++++ 6 files changed, 522 insertions(+), 16 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java index 8da9a34968..a676b7e487 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java @@ -29,6 +29,7 @@ */ final class AffixCondition { + /** The shared instance for the condition {@code .}, which accepts every stem. */ private static final AffixCondition ANY = new AffixCondition(new char[0][], null, true); /** Per position: the accepted characters, or {@code null} for any character. */ @@ -44,7 +45,9 @@ private AffixCondition(char[][] accepted, boolean[] negated, boolean suffix) { } /** - * Parses a condition field. + * Parses a condition field. Each pattern position is a literal character, a + * {@code .} matching any character, or a bracketed class such as {@code [sx]}; a + * class starting with {@code ^} is negated and matches any character outside it. * * @param pattern The condition text from the affix rule. * @param suffix Whether the owning rule is a suffix rule. @@ -95,7 +98,9 @@ static AffixCondition parse(String pattern, boolean suffix, int lineNumber) } /** - * Tests a candidate stem against the condition at its anchored side. + * Tests a candidate stem against the condition at its anchored side: the last + * positions of the stem for a suffix condition, the first positions for a prefix + * condition. A stem shorter than the condition never matches. * * @param stem The candidate stem after affix removal and strip restoration. * @return {@code true} if the stem satisfies the condition. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 02dff67559..ea8f623397 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -54,8 +54,13 @@ */ public final class HunspellDictionary { - /** One parsed affix rule; {@code affix} is the surface material added to the stem, - * and {@code continuation} lists the flags of affixes that may stack on top. */ + /** + * One parsed affix rule. {@code affix} is the surface material the rule adds to the + * stem, {@code strip} is the stem material the rule replaces (restored during + * analysis), {@code crossProduct} states whether the rule may combine with an affix + * of the opposite kind, and {@code continuation} lists the flags of further affixes + * that may stack on top of this one. + */ record Affix(int flag, boolean crossProduct, String strip, String affix, AffixCondition condition, int[] continuation) { @@ -169,6 +174,13 @@ static boolean hasFlag(List flagSets, int flag) { return false; } + /** + * Reads a stream fully into memory. The stream is not closed. + * + * @param in The stream to drain. + * @return All bytes the stream produced. Never {@code null}. + * @throws IOException Thrown if reading fails. + */ private static byte[] readAll(InputStream in) throws IOException { final ByteArrayOutputStream out = new ByteArrayOutputStream(); final byte[] buffer = new byte[8192]; @@ -179,7 +191,15 @@ private static byte[] readAll(InputStream in) throws IOException { return out.toByteArray(); } - /** Finds the {@code SET} declaration by scanning the raw bytes as ASCII. */ + /** + * Finds the {@code SET} declaration by scanning the raw affix bytes as ASCII, which + * is safe because the declaration itself is ASCII in every supported encoding. Both + * files are then decoded with the declared charset. + * + * @param affixBytes The raw affix file content. + * @return The declared charset, or UTF-8 when no declaration is present. + * @throws IOException Thrown if the declared encoding name is not supported. + */ private static Charset declaredCharset(byte[] affixBytes) throws IOException { final String ascii = new String(affixBytes, StandardCharsets.US_ASCII); for (final String line : splitLines(ascii)) { @@ -196,9 +216,14 @@ private static Charset declaredCharset(byte[] affixBytes) throws IOException { return StandardCharsets.UTF_8; } - /** The flag encodings a dictionary may declare. */ + /** The flag encodings a dictionary may declare with the {@code FLAG} directive. */ private enum FlagMode { - CHAR, LONG, NUM + /** The default: each single character is one flag. */ + CHAR, + /** Declared as {@code FLAG long}: each pair of characters is one flag. */ + LONG, + /** Declared as {@code FLAG num}: comma-separated decimal numbers are flags. */ + NUM } /** The parsed affix file content. */ @@ -208,6 +233,16 @@ private static final class AffixFile { private FlagMode flagMode = FlagMode.CHAR; } + /** + * Parses the affix file: the {@code FLAG} declaration and the {@code PFX} and + * {@code SFX} blocks. Directives outside the supported set (compounding, + * conversion tables, suggestion options, ...) are skipped, so their rules never + * fire and unsupported analyses are missed rather than invented. + * + * @param content The decoded affix file content. + * @return The parsed rules and flag mode. Never {@code null}. + * @throws IOException Thrown if a supported directive is malformed. + */ private static AffixFile parseAffix(String content) throws IOException { final AffixFile result = new AffixFile(); final String[] lines = splitLines(content); @@ -243,7 +278,18 @@ private static AffixFile parseAffix(String content) throws IOException { return result; } - /** Parses one PFX or SFX header and its rule lines; returns the next line index. */ + /** + * Parses one {@code PFX} or {@code SFX} block: the header line naming the flag, the + * cross-product marker, and the rule count, followed by exactly that many rule + * lines. + * + * @param lines All lines of the affix file. + * @param index The line index of the block header. + * @param header The already-split header fields. + * @param result The parse target the rules are added to. + * @return The index of the first line after the block. + * @throws IOException Thrown if the header or a rule line is malformed. + */ private static int parseAffixBlock(String[] lines, int index, String[] header, AffixFile result) throws IOException { if (header.length < 4) { @@ -289,6 +335,17 @@ private static int parseAffixBlock(String[] lines, int index, String[] header, return line; } + /** + * Parses the word list: an optional leading entry count, then one entry per line + * consisting of the word, an optional {@code /flags} run, and optional + * whitespace-separated morphological fields, which are ignored. A slash escaped as + * {@code \/} belongs to the word itself and is unescaped in the stored key. + * + * @param content The decoded word-list content. + * @param flagMode The flag encoding declared by the affix file. + * @return The words mapped to the flag sets of their entries. Never {@code null}. + * @throws IOException Thrown if a flag run is malformed. + */ private static Map> parseWordList(String content, FlagMode flagMode) throws IOException { final String[] lines = splitLines(content); @@ -325,6 +382,13 @@ private static Map> parseWordList(String content, return entries; } + /** + * Checks whether a line consists purely of decimal digits, which identifies the + * optional entry-count header of a word list. + * + * @param line The trimmed line to inspect. + * @return {@code true} if the line is a non-empty digit run. + */ private static boolean isCount(String line) { if (line.isEmpty()) { return false; @@ -337,7 +401,13 @@ private static boolean isCount(String line) { return true; } - /** Finds the first {@code /} that is not escaped as {@code \/}. */ + /** + * Finds the first {@code /} that is not escaped as {@code \/}, which separates the + * word from its flag run in a word-list entry. + * + * @param line The word-list line to scan. + * @return The index of the separator, or {@code -1} when the entry has no flags. + */ private static int unescapedSlash(String line) { for (int i = 0; i < line.length(); i++) { if (line.charAt(i) == '/' && (i == 0 || line.charAt(i - 1) != '\\')) { @@ -347,6 +417,13 @@ private static int unescapedSlash(String line) { return -1; } + /** + * Finds the first whitespace character, which terminates the word or flag field of + * a word-list entry before its optional morphological fields. + * + * @param text The text to scan. + * @return The index of the first whitespace character, or {@code -1} if none. + */ private static int whitespaceIndex(String text) { for (int i = 0; i < text.length(); i++) { if (StringUtil.isWhitespace(text.charAt(i))) { @@ -356,6 +433,17 @@ private static int whitespaceIndex(String text) { return -1; } + /** + * Parses a flag run according to the declared flag mode: single characters in + * {@code char} mode, character pairs packed into one {@code int} in {@code long} + * mode, and comma-separated decimal numbers in {@code num} mode. + * + * @param text The flag run without its leading {@code /}. + * @param mode The declared flag encoding. + * @param lineNumber The source line, for error messages. + * @return The parsed flags. Never {@code null}. + * @throws IOException Thrown if the run does not fit the declared encoding. + */ private static int[] parseFlags(String text, FlagMode mode, int lineNumber) throws IOException { switch (mode) { @@ -391,6 +479,16 @@ private static int[] parseFlags(String text, FlagMode mode, int lineNumber) } } + /** + * Parses a field that must contain exactly one flag, such as the flag name in an + * affix block header. + * + * @param text The flag field. + * @param mode The declared flag encoding. + * @param lineNumber The source line, for error messages. + * @return The single parsed flag. + * @throws IOException Thrown if the field holds no flag or more than one. + */ private static int parseFlag(String text, FlagMode mode, int lineNumber) throws IOException { final int[] flags = parseFlags(text, mode, lineNumber); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index ba2dd14eb6..0639e4ea2d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -34,10 +34,12 @@ *

{@link #stem(CharSequence)} returns the first analysis, preferring the word's own * dictionary entry; {@link #stemAll(CharSequence)} returns every distinct analysis. A * word with no analysis is returned unchanged, so the stemmer degrades to identity on - * unknown vocabulary. Capitalized forms also try their lowercase variant.

+ * unknown vocabulary. A form containing uppercase characters is also analyzed in its + * lowercase variant, so sentence-initial capitalization does not hide an entry.

* - *

The stemmer reads only immutable dictionary state and is safe to share between - * threads, satisfying the single-thread confinement contract trivially.

+ *

The {@link Stemmer} interface leaves thread safety to the implementation. This + * implementation reads only the immutable dictionary state, so a single instance is + * safe to share between threads.

* * @since 3.0.0 */ @@ -80,13 +82,29 @@ public List stemAll(CharSequence word) { return List.copyOf(new ArrayList(analyses)); } - /** Collects the case variants to analyze: the surface form, then its lowercase. */ + /** + * Collects the case variants to analyze: the surface form first, then its lowercase + * form when the two differ. Ordering matters because the first analysis found wins + * in {@link #stem(CharSequence)}. + * + * @param surface The surface form. + * @return The variants in analysis order. Never {@code null} or empty. + */ private static List variants(String surface) { final String lowered = StringUtil.toLowerCase(surface); return lowered.equals(surface) ? List.of(surface) : List.of(surface, lowered); } - /** Runs direct lookup, suffix removal, prefix removal, and cross products. */ + /** + * Adds every analysis of one case variant to the result set: the word's own + * dictionary entry, single suffix removal, twofold suffix removal through + * continuation classes, single prefix removal, and cross-product removal of one + * prefix together with one suffix. Insertion order into the set fixes the + * preference order reported by {@link #stemAll(CharSequence)}. + * + * @param word The case variant to analyze. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ private void analyze(String word, Set analyses) { if (dictionary.lookup(word) != null) { analyses.add(word); @@ -144,7 +162,10 @@ private void analyze(String word, Set analyses) { } /** - * Undoes one suffix rule. + * Undoes one suffix rule: cuts the affix material off the end of the word, restores + * the strip string the rule removed on application, and checks the rule's condition + * against the restored stem. Rules with empty affix material and candidates that + * would leave an empty stem are rejected. * * @param word The surface form. * @param suffix The rule to undo. @@ -161,7 +182,10 @@ private static String removeSuffix(String word, Affix suffix) { } /** - * Undoes one prefix rule. + * Undoes one prefix rule: cuts the affix material off the start of the word, + * restores the strip string the rule removed on application, and checks the rule's + * condition against the restored stem. Rules with empty affix material and + * candidates that would leave an empty stem are rejected. * * @param word The surface form. * @param prefix The rule to undo. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java index eb0a8d596e..21d66be2b0 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -45,6 +45,10 @@ public HunspellStemmerFactory(HunspellDictionary dictionary) { this.dictionary = dictionary; } + /** + * {@return a new {@link HunspellStemmer} over the shared dictionary} Every call + * creates a fresh instance; all instances read the same immutable dictionary. + */ @Override public Stemmer newStemmer() { return new HunspellStemmer(dictionary); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java new file mode 100644 index 0000000000..e19ce10729 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.stemmer.Stemmer; + +/** + * Demonstrates the intended end-to-end usage of the Hunspell stemming classes: a user + * writes (or ships) a {@code .aff}/{@code .dic} file pair, loads it once into a + * {@link HunspellDictionary}, wraps the dictionary in a {@link HunspellStemmerFactory}, + * and obtains {@link Stemmer} instances from the factory wherever stemming is needed. + * The fixture dictionary is authored inside this test class, so no external dictionary + * data is involved. + */ +public class HunspellStemmerFactoryTest { + + /** + * The affix fixture: the prefix {@code re-}, the suffix {@code -er} whose continuation + * class {@code S} lets the plural {@code -s} stack on top of it, and the plural + * {@code -s} itself, restricted to stems not ending in {@code s}, {@code x}, or + * {@code y}. All three rules opt into cross-product combination. + */ + private static final String AFFIX = String.join("\n", + "# project-authored test fixture", + "SET UTF-8", + "", + "PFX R Y 1", + "PFX R 0 re .", + "", + "SFX E Y 1", + "SFX E 0 er/S .", + "", + "SFX S Y 1", + "SFX S 0 s [^sxy]", + ""); + + /** + * The word-list fixture: {@code work} accepts the prefix and both suffixes, + * {@code paint} accepts only the agentive {@code -er}. + */ + private static final String WORDS = String.join("\n", + "2", + "work/RES", + "paint/E", + ""); + + /** + * Writes the fixture dictionary pair into a directory and loads it through the + * file-based {@link HunspellDictionary#load(Path, Path)} entry point. + * + * @param directory The directory to write into. Must not be {@code null} and must + * denote an existing directory. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if writing or loading fails. + * @throws IllegalArgumentException Thrown if {@code directory} is unusable. + */ + private static HunspellDictionary writeAndLoadFixture(Path directory) throws IOException { + if (directory == null || !Files.isDirectory(directory)) { + throw new IllegalArgumentException("directory must be an existing directory"); + } + final Path affixFile = directory.resolve("fixture.aff"); + final Path dictionaryFile = directory.resolve("fixture.dic"); + Files.write(affixFile, AFFIX.getBytes(StandardCharsets.UTF_8)); + Files.write(dictionaryFile, WORDS.getBytes(StandardCharsets.UTF_8)); + return HunspellDictionary.load(affixFile, dictionaryFile); + } + + /** + * Walks the whole intended flow on a single thread: files on disk, one dictionary, + * one factory, one stemmer, and exact stems for a prefixed form, a suffixed form, a + * twofold suffix chain, a cross-product form, an in-dictionary word, and an unknown + * word. + * + * @param tempDir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or loaded. + */ + @Test + void testEndToEndUsageFromFiles(@TempDir Path tempDir) throws IOException { + final HunspellDictionary dictionary = writeAndLoadFixture(tempDir); + final HunspellStemmerFactory factory = new HunspellStemmerFactory(dictionary); + final Stemmer stemmer = factory.newStemmer(); + + // one suffix removed + Assertions.assertEquals("work", stemmer.stem("worker").toString()); + Assertions.assertEquals("paint", stemmer.stem("painter").toString()); + // twofold suffixes: -s stacks on -er through the continuation class S + Assertions.assertEquals("work", stemmer.stem("workers").toString()); + // one prefix removed + Assertions.assertEquals("work", stemmer.stem("rework").toString()); + // cross product: the prefix re- and the suffix -s on the same stem + Assertions.assertEquals("work", stemmer.stem("reworks").toString()); + // a word that is itself listed stems to itself + Assertions.assertEquals("work", stemmer.stem("work").toString()); + // unknown vocabulary passes through unchanged + Assertions.assertEquals("table", stemmer.stem("table").toString()); + } + + /** + * Shares one factory between two threads: each thread obtains its own stemmer + * instance from the factory and stems the same inputs. The test asserts that the two + * instances are distinct objects and that their results are identical to each other + * and to the expected stems. + * + * @param tempDir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture cannot be loaded or a worker fails. + */ + @Test + void testFactorySharedAcrossThreads(@TempDir Path tempDir) throws Exception { + final HunspellStemmerFactory factory = + new HunspellStemmerFactory(writeAndLoadFixture(tempDir)); + final List inputs = List.of("workers", "reworks", "painter", "table"); + final List expected = List.of("work", "work", "paint", "table"); + + final Stemmer[] created = new Stemmer[2]; + final ExecutorService pool = Executors.newFixedThreadPool(2); + try { + final List>> futures = new ArrayList<>(2); + for (int worker = 0; worker < 2; worker++) { + final int slot = worker; + futures.add(pool.submit(() -> { + final Stemmer stemmer = factory.newStemmer(); + created[slot] = stemmer; + final List stems = new ArrayList<>(inputs.size()); + for (final String input : inputs) { + stems.add(stemmer.stem(input).toString()); + } + return stems; + })); + } + final List first = futures.get(0).get(); + final List second = futures.get(1).get(); + Assertions.assertEquals(expected, first); + Assertions.assertEquals(expected, second); + } finally { + pool.shutdownNow(); + } + Assertions.assertNotSame(created[0], created[1]); + } + + /** + * Verifies that the file-based entry point rejects {@code null} paths with the + * documented exception instead of failing later with an obscure error. + */ + @Test + void testNullPathsAreRejected() { + final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load((Path) null, (Path) null)); + Assertions.assertEquals("affixFile and dictionaryFile must not be null", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 2045f6e645..c6304ab156 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.Assertions; @@ -179,6 +180,201 @@ void testFactoryHandsOutWorkingStemmers() throws IOException { Assertions.assertEquals("pony", fresh.stem("ponies").toString()); } + /** + * Loads a dictionary from in-memory affix and word-list content, both encoded as + * UTF-8, through the stream-based entry point. + * + * @param affix The {@code .aff} content. Must not be {@code null}. + * @param words The {@code .dic} content. Must not be {@code null}. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if the content is malformed. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + private static HunspellDictionary load(String affix, String words) throws IOException { + if (affix == null || words == null) { + throw new IllegalArgumentException("affix and words must not be null"); + } + return HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + } + + /** + * Verifies that cross-product combination of a prefix with a suffix only happens + * when both rules declare the cross-product marker {@code Y}. Removing just the one + * affix whose rule exists keeps working; the combined form must not be analyzed. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCrossProductRequiresBothRulesOptIn() throws IOException { + // the prefix rule declares N, so it never combines with the suffix + final HunspellStemmer prefixOptedOut = new HunspellStemmer(load(String.join("\n", + "PFX U N 1", + "PFX U 0 un .", + "SFX S Y 1", + "SFX S 0 s .", + ""), "1\nlock/US\n")); + Assertions.assertEquals("lock", prefixOptedOut.stem("unlock").toString()); + Assertions.assertEquals("lock", prefixOptedOut.stem("locks").toString()); + Assertions.assertEquals("unlocks", prefixOptedOut.stem("unlocks").toString()); + + // the suffix rule declares N, so the combined form is likewise not analyzed + final HunspellStemmer suffixOptedOut = new HunspellStemmer(load(String.join("\n", + "PFX U Y 1", + "PFX U 0 un .", + "SFX S N 1", + "SFX S 0 s .", + ""), "1\nlock/US\n")); + Assertions.assertEquals("lock", suffixOptedOut.stem("unlock").toString()); + Assertions.assertEquals("lock", suffixOptedOut.stem("locks").toString()); + Assertions.assertEquals("unlocks", suffixOptedOut.stem("unlocks").toString()); + } + + /** + * Verifies that a non-negated character class rejects a candidate stem: the + * {@code es} rule requires a stem ending in {@code s} or {@code x}, so removing + * {@code es} from {@code cates} produces {@code cat}, which the class rejects, and + * the surface form falls through unchanged. + */ + @Test + void testPositiveCharacterClassRejectsCandidate() { + Assertions.assertEquals("cates", stemmer.stem("cates").toString()); + Assertions.assertEquals(1, stemmer.stemAll("cates").size()); + } + + /** + * Verifies that the {@code SET} declaration selects the charset both files are + * decoded with: a word list holding the byte {@code 0xE9} only maps to the word + * caf\u00E9 (e with acute accent) when decoded as ISO-8859-1, as the affix file declares. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testSetDeclarationSelectsEncoding() throws IOException { + final Charset latin1 = StandardCharsets.ISO_8859_1; + final String affix = String.join("\n", + "SET ISO8859-1", + "SFX S Y 1", + "SFX S 0 s .", + ""); + final String words = "1\ncaf\u00E9/S\n"; + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(affix.getBytes(latin1)), + new ByteArrayInputStream(words.getBytes(latin1))); + final HunspellStemmer latin1Stemmer = new HunspellStemmer(dictionary); + Assertions.assertEquals("caf\u00E9", latin1Stemmer.stem("caf\u00E9s").toString()); + Assertions.assertEquals("caf\u00E9", latin1Stemmer.stem("caf\u00E9").toString()); + } + + /** + * Verifies that continuation classes also work in {@code FLAG long} mode, where a + * flag is a two-character run: the plural {@code Bb} stacks on the agentive + * {@code Aa} to analyze a twofold suffix chain. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testLongFlagContinuation() throws IOException { + final HunspellStemmer longFlags = new HunspellStemmer(load(String.join("\n", + "FLAG long", + "SFX Aa Y 1", + "SFX Aa 0 er/Bb .", + "SFX Bb Y 1", + "SFX Bb 0 s .", + ""), "1\nkind/Aa\n")); + Assertions.assertEquals("kind", longFlags.stem("kinder").toString()); + Assertions.assertEquals("kind", longFlags.stem("kinders").toString()); + } + + /** + * Verifies that cross-product prefix and suffix combination also works in + * {@code FLAG num} mode, where flags are comma-separated decimal numbers. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testNumericFlagCrossProduct() throws IOException { + final HunspellStemmer numericFlags = new HunspellStemmer(load(String.join("\n", + "FLAG num", + "PFX 1 Y 1", + "PFX 1 0 un .", + "SFX 2 Y 1", + "SFX 2 0 s .", + ""), "1\nlock/1,2\n")); + Assertions.assertEquals("lock", numericFlags.stem("unlock").toString()); + Assertions.assertEquals("lock", numericFlags.stem("locks").toString()); + Assertions.assertEquals("lock", numericFlags.stem("unlocks").toString()); + } + + /** + * Verifies the exact exception and message for each malformed {@code FLAG} + * declaration the parser detects: a missing mode and an unrecognized mode name. + */ + @Test + void testMalformedFlagDeclarationMessages() { + IOException e = Assertions.assertThrows(IOException.class, + () -> load("FLAG\n", "0\n")); + Assertions.assertEquals("FLAG line without a mode at line 1", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, () -> load("FLAG short\n", "0\n")); + Assertions.assertEquals("unsupported FLAG mode 'short' at line 1", e.getMessage()); + } + + /** + * Verifies the exact exception and message for each malformed affix block the + * parser detects: a header with too few fields, a non-numeric rule count, a block + * with fewer rule lines than its count announces, a rule line whose type tag does + * not match its header, and an unterminated character class in a condition. + */ + @Test + void testMalformedAffixBlockMessages() { + IOException e = Assertions.assertThrows(IOException.class, + () -> load("PFX U Y\n", "0\n")); + Assertions.assertEquals("malformed affix header at line 1", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y many\nSFX S 0 s .\n", "0\n")); + Assertions.assertEquals("malformed affix rule count at line 1", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y 2\nSFX S 0 s .", "0\n")); + Assertions.assertEquals("affix block truncated at line 3", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y 1\nPFX S 0 s .\n", "0\n")); + Assertions.assertEquals("malformed affix rule at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SFX S Y 1\nSFX S 0 s [ab\n", "0\n")); + Assertions.assertEquals("unterminated character class at line 2", e.getMessage()); + } + + /** + * Verifies the exact exception and message for each malformed flag value the parser + * detects: an odd-length flag run in {@code FLAG long} mode, a non-numeric flag in + * {@code FLAG num} mode, an affix header naming more than one flag, and a + * {@code SET} declaration naming an unknown encoding. + */ + @Test + void testMalformedFlagValueMessages() { + IOException e = Assertions.assertThrows(IOException.class, + () -> load("FLAG long\n", "1\nwalk/AaB\n")); + Assertions.assertEquals("odd long-flag run at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("FLAG num\n", "1\nwalk/12,x\n")); + Assertions.assertEquals("malformed numeric flag at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("FLAG long\nSFX AaBb Y 1\nSFX AaBb 0 s .\n", "0\n")); + Assertions.assertEquals("expected exactly one flag at line 2", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, + () -> load("SET NO-SUCH-ENCODING\n", "0\n")); + Assertions.assertEquals("unsupported SET encoding: NO-SUCH-ENCODING", e.getMessage()); + } + @Test void testMalformedInputFailsLoud() { Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( From bc718eed437464850a62305185d3fb744182359d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 01:45:20 -0400 Subject: [PATCH 04/24] OPENNLP-1893: Document Hunspell dictionary acquisition with a license-preserving download helper --- .../dev/README-hunspell-dictionaries.md | 53 +++++++++++++++ .../dev/download-hunspell-dictionary.sh | 66 +++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md create mode 100755 opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh diff --git a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md new file mode 100644 index 0000000000..98195b029d --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md @@ -0,0 +1,53 @@ + + +# Hunspell dictionaries for the affix stemmer + +The Hunspell stemmer (`opennlp.tools.stemmer.hunspell`) is a clean-room engine over the documented Hunspell dictionary format: a `.dic` word list plus its `.aff` affix companion, both supplied by the user. Apache OpenNLP bundles no dictionary data, so the dictionaries' own licenses never attach to the library; whichever dictionary you download, its license is stated in the readme shipped alongside it and is yours to comply with. + +## Where dictionaries come from + +The LibreOffice project maintains a large collection of Hunspell dictionaries, one directory per language, at `github.com/LibreOffice/dictionaries`. Licenses differ per dictionary, which is exactly why nothing is bundled: for example, the `en_US` dictionary derives from SCOWL and states its terms in `README_en_US.txt` in the same directory. Many other sources work too; the engine only cares that the pair follows the Hunspell format. + +The helper next to this file fetches a pair together with its readme files: + +``` +./download-hunspell-dictionary.sh en en_US /tmp/hunspell-en_US +``` + +## Loading and stemming + +```java +import java.nio.file.Path; +import opennlp.tools.stemmer.Stemmer; +import opennlp.tools.stemmer.hunspell.HunspellDictionary; +import opennlp.tools.stemmer.hunspell.HunspellStemmerFactory; + +HunspellDictionary dictionary = HunspellDictionary.load( + Path.of("/tmp/hunspell-en_US/en_US.aff"), + Path.of("/tmp/hunspell-en_US/en_US.dic")); +HunspellStemmerFactory factory = new HunspellStemmerFactory(dictionary); + +Stemmer stemmer = factory.newStemmer(); +CharSequence stem = stemmer.stem("workers"); +``` + +The dictionary is immutable and safe to share between threads; the factory hands out a fresh stemmer per call, so each thread takes its own from `newStemmer()`. A dictionary that declares a non-UTF-8 encoding through the `SET` directive in its `.aff` file is decoded accordingly; nothing needs converting beforehand. + +## What the engine supports + +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `long`, and `num`, and the `SET` encoding declaration. Compounding and conversion tables are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. diff --git a/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh b/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh new file mode 100755 index 0000000000..5ed95511b0 --- /dev/null +++ b/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Fetches one Hunspell dictionary pair (.aff and .dic) plus its license/readme files +# from the LibreOffice dictionaries collection. Each dictionary carries its own +# license, stated in the readme files this script downloads alongside it, and you +# accept that license by using the dictionary. Apache OpenNLP bundles no dictionary +# data. See README-hunspell-dictionaries.md in this directory for the Java steps that +# follow. + +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 + echo "" >&2 + echo " collection-dir the language directory inside the LibreOffice dictionaries" >&2 + echo " repository, for example: en" >&2 + echo " dictionary-name the dictionary base name inside it, for example: en_US" >&2 + echo " target-dir where the .aff, .dic, and readme files are placed" >&2 + echo "" >&2 + echo "example: $0 en en_US /tmp/hunspell-en_US" >&2 + exit 2 +} + +[ $# -ne 3 ] && usage +collection="$1" +name="$2" +target="$3" + +base="https://raw.githubusercontent.com/LibreOffice/dictionaries/master/${collection}" +mkdir -p "${target}" + +# The .aff and .dic pair is mandatory; a missing file fails the script. +for ext in aff dic; do + echo "downloading ${name}.${ext}" + curl --fail --location --silent --show-error --retry 3 \ + --output "${target}/${name}.${ext}" "${base}/${name}.${ext}" +done + +# The readme carries the dictionary's license; keep it next to the data. Different +# collections name it differently, so try the common patterns and keep what exists. +for readme in "README_${name}.txt" "README_${collection}.txt" "README.txt" "license.txt"; do + if curl --fail --location --silent --retry 3 \ + --output "${target}/${readme}" "${base}/${readme}" 2>/dev/null; then + echo "downloaded ${readme} (contains the dictionary's license; read it)" + else + rm -f "${target}/${readme}" + fi +done + +echo "" +echo "stored ${target}/${name}.aff and ${target}/${name}.dic" +echo "next: load them from Java; see README-hunspell-dictionaries.md" From 322fd1a5aedb59fbaf0cba1104eb9a0801aae6f9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 22:36:16 -0400 Subject: [PATCH 05/24] OPENNLP-1893: Cut morphology like hunspell does, accept UTF-8 flags and strip-only rules, and read the parser through the whitespace seam --- .../dev/README-hunspell-dictionaries.md | 4 +- .../stemmer/hunspell/HunspellDictionary.java | 109 ++++++++----- .../stemmer/hunspell/HunspellStemmer.java | 27 ++-- .../stemmer/hunspell/HunspellStemmerTest.java | 144 ++++++++++++++++++ 4 files changed, 237 insertions(+), 47 deletions(-) diff --git a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md index 98195b029d..31edf4fe0b 100644 --- a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md +++ b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md @@ -46,8 +46,10 @@ Stemmer stemmer = factory.newStemmer(); CharSequence stem = stemmer.stem("workers"); ``` +What `stem` evaluates to is decided by the dictionary you loaded, and this project ships no dictionary data, so no result is claimed here for `en_US`. What is verified is the flow above: `HunspellStemmerFactoryTest#testEndToEndUsageFromFiles` runs exactly these calls against a project-authored `.aff`/`.dic` pair written to disk, in which `work` carries the agentive `-er` rule and its continuation class for the plural `-s`, and asserts that `stemmer.stem("workers")` returns `work`. + The dictionary is immutable and safe to share between threads; the factory hands out a fresh stemmer per call, so each thread takes its own from `newStemmer()`. A dictionary that declares a non-UTF-8 encoding through the `SET` directive in its `.aff` file is decoded accordingly; nothing needs converting beforehand. ## What the engine supports -Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `long`, and `num`, and the `SET` encoding declaration. Compounding and conversion tables are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, and the `SET` encoding declaration. Compounding and conversion tables are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index ea8f623397..6dbb001ebe 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -41,8 +41,9 @@ *

Supported affix features: {@code PFX} and {@code SFX} rules with strip strings, * character-class conditions, and cross-product combination of one prefix with one * suffix; twofold suffixes through the continuation classes on suffix rules; - * {@code FLAG} modes {@code char} (default), {@code long}, and {@code num}; the - * {@code SET} encoding declaration. Compounding and conversion tables are not + * {@code FLAG} modes {@code char} (default), {@code UTF-8}, {@code long}, and + * {@code num}; the {@code SET} encoding declaration. Compounding and conversion tables + * are not * interpreted in this version; rules using them simply do not fire, so unsupported * analyses are missed rather than invented.

* @@ -203,9 +204,9 @@ private static byte[] readAll(InputStream in) throws IOException { private static Charset declaredCharset(byte[] affixBytes) throws IOException { final String ascii = new String(affixBytes, StandardCharsets.US_ASCII); for (final String line : splitLines(ascii)) { - final String trimmed = line.trim(); + final String trimmed = trim(line); if (trimmed.startsWith("SET ") || trimmed.startsWith("SET\t")) { - final String name = trimmed.substring(4).trim(); + final String name = trim(trimmed.substring(4)); try { return Charset.forName(name); } catch (RuntimeException e) { @@ -218,7 +219,11 @@ private static Charset declaredCharset(byte[] affixBytes) throws IOException { /** The flag encodings a dictionary may declare with the {@code FLAG} directive. */ private enum FlagMode { - /** The default: each single character is one flag. */ + /** + * The default: each single character is one flag. Also what {@code FLAG UTF-8} + * declares, which asks for single-character flags in a file the {@code SET} + * declaration already had decoded. + */ CHAR, /** Declared as {@code FLAG long}: each pair of characters is one flag. */ LONG, @@ -261,6 +266,7 @@ private static AffixFile parseAffix(String content) throws IOException { result.flagMode = switch (fields[1]) { case "long" -> FlagMode.LONG; case "num" -> FlagMode.NUM; + case "UTF-8" -> FlagMode.CHAR; default -> throw new IOException( "unsupported FLAG mode '" + fields[1] + "' at line " + (i + 1)); }; @@ -337,9 +343,11 @@ private static int parseAffixBlock(String[] lines, int index, String[] header, /** * Parses the word list: an optional leading entry count, then one entry per line - * consisting of the word, an optional {@code /flags} run, and optional - * whitespace-separated morphological fields, which are ignored. A slash escaped as - * {@code \/} belongs to the word itself and is unescaped in the stored key. + * consisting of the word, an optional {@code /flags} run, and optional trailing + * morphological fields, which are ignored. The morphological fields are cut off + * first, because the flag separator is only meaningful in what precedes them; a word + * may itself contain spaces. A slash escaped as {@code \/} belongs to the word itself + * and is unescaped in the stored key. * * @param content The decoded word-list content. * @param flagMode The flag encoding declared by the affix file. @@ -351,30 +359,22 @@ private static Map> parseWordList(String content, final String[] lines = splitLines(content); final Map> entries = new HashMap<>(); int start = 0; - if (lines.length > 0 && isCount(lines[0].trim())) { + if (lines.length > 0 && isCount(trim(lines[0]))) { start = 1; } for (int i = start; i < lines.length; i++) { - final String line = lines[i].trim(); + final String line = trim(lines[i]); if (line.isEmpty()) { continue; } - String word = line; + final int morphology = morphologyIndex(line); + final String entry = morphology < 0 ? line : trim(line.substring(0, morphology)); + String word = entry; int[] flags = new int[0]; - final int slash = unescapedSlash(line); + final int slash = unescapedSlash(entry); if (slash >= 0) { - word = line.substring(0, slash); - String flagText = line.substring(slash + 1); - final int fieldEnd = whitespaceIndex(flagText); - if (fieldEnd >= 0) { - flagText = flagText.substring(0, fieldEnd); - } - flags = parseFlags(flagText, flagMode, i + 1); - } else { - final int fieldEnd = whitespaceIndex(word); - if (fieldEnd >= 0) { - word = word.substring(0, fieldEnd); - } + word = entry.substring(0, slash); + flags = parseFlags(entry.substring(slash + 1), flagMode, i + 1); } entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) .add(flags); @@ -418,19 +418,54 @@ private static int unescapedSlash(String line) { } /** - * Finds the first whitespace character, which terminates the word or flag field of - * a word-list entry before its optional morphological fields. + * Finds where the trailing morphological fields of a word-list entry begin, which + * terminates the word and its flag run. A morphological field is either introduced by + * a tabulator, the older separator, or written as a two-letter tag followed by + * {@code :} and preceded by whitespace, such as {@code po:verb}. Whitespace that is + * not followed by such a tag belongs to the word, because a word-list entry may name + * several words. * - * @param text The text to scan. - * @return The index of the first whitespace character, or {@code -1} if none. + * @param line The trimmed word-list line to scan. + * @return The index at which the morphological fields begin, or {@code -1} if the + * entry carries none. */ - private static int whitespaceIndex(String text) { - for (int i = 0; i < text.length(); i++) { - if (StringUtil.isWhitespace(text.charAt(i))) { - return i; + private static int morphologyIndex(String line) { + int cut = -1; + for (int i = 4; i < line.length(); i++) { + if (line.charAt(i) == ':' && StringUtil.isWhitespace(line.charAt(i - 3))) { + int fieldStart = i - 3; + while (fieldStart > 0 && StringUtil.isWhitespace(line.charAt(fieldStart - 1))) { + fieldStart--; + } + // a tag with no word in front of it is not a morphological field + cut = fieldStart == 0 ? -1 : fieldStart; + break; } } - return -1; + final int tab = line.indexOf('\t'); + if (tab >= 0 && (cut < 0 || tab < cut)) { + cut = tab; + } + return cut; + } + + /** + * Removes leading and trailing whitespace, using the whitespace definition the rest + * of the parser scans with. + * + * @param text The text to trim. + * @return The text without leading or trailing whitespace. Never {@code null}. + */ + private static String trim(String text) { + int start = 0; + int end = text.length(); + while (start < end && StringUtil.isWhitespace(text.charAt(start))) { + start++; + } + while (end > start && StringUtil.isWhitespace(text.charAt(end - 1))) { + end--; + } + return text.substring(start, end); } /** @@ -438,7 +473,8 @@ private static int whitespaceIndex(String text) { * {@code char} mode, character pairs packed into one {@code int} in {@code long} * mode, and comma-separated decimal numbers in {@code num} mode. * - * @param text The flag run without its leading {@code /}. + * @param text The flag run without its leading {@code /}. An empty run carries no + * flags in every mode. * @param mode The declared flag encoding. * @param lineNumber The source line, for error messages. * @return The parsed flags. Never {@code null}. @@ -446,13 +482,16 @@ private static int whitespaceIndex(String text) { */ private static int[] parseFlags(String text, FlagMode mode, int lineNumber) throws IOException { + if (text.isEmpty()) { + return new int[0]; + } switch (mode) { case NUM: { final String[] parts = splitOn(text, ','); final int[] flags = new int[parts.length]; for (int i = 0; i < parts.length; i++) { try { - flags[i] = Integer.parseInt(parts[i].trim()); + flags[i] = Integer.parseInt(trim(parts[i])); } catch (NumberFormatException e) { throw new IOException("malformed numeric flag at line " + lineNumber, e); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 0639e4ea2d..eb55bb6d8e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -164,39 +164,44 @@ private void analyze(String word, Set analyses) { /** * Undoes one suffix rule: cuts the affix material off the end of the word, restores * the strip string the rule removed on application, and checks the rule's condition - * against the restored stem. Rules with empty affix material and candidates that - * would leave an empty stem are rejected. + * against the restored stem. A strip-only rule, whose affix material is empty, is + * undone by restoring its strip string alone. Rules that neither add nor remove + * material and candidates that would leave an empty stem are rejected. * * @param word The surface form. * @param suffix The rule to undo. * @return The candidate stem, or {@code null} when the rule does not apply. */ private static String removeSuffix(String word, Affix suffix) { - if (suffix.affix().isEmpty() || !word.endsWith(suffix.affix()) - || word.length() - suffix.affix().length() + suffix.strip().length() == 0) { + final String affix = suffix.affix(); + final String strip = suffix.strip(); + if (affix.isEmpty() && strip.isEmpty() || !word.endsWith(affix) + || word.length() - affix.length() + strip.length() == 0) { return null; } - final String stem = - word.substring(0, word.length() - suffix.affix().length()) + suffix.strip(); + final String stem = word.substring(0, word.length() - affix.length()) + strip; return suffix.condition().matches(stem) ? stem : null; } /** * Undoes one prefix rule: cuts the affix material off the start of the word, * restores the strip string the rule removed on application, and checks the rule's - * condition against the restored stem. Rules with empty affix material and - * candidates that would leave an empty stem are rejected. + * condition against the restored stem. A strip-only rule, whose affix material is + * empty, is undone by restoring its strip string alone. Rules that neither add nor + * remove material and candidates that would leave an empty stem are rejected. * * @param word The surface form. * @param prefix The rule to undo. * @return The candidate stem, or {@code null} when the rule does not apply. */ private static String removePrefix(String word, Affix prefix) { - if (prefix.affix().isEmpty() || !word.startsWith(prefix.affix()) - || word.length() - prefix.affix().length() + prefix.strip().length() == 0) { + final String affix = prefix.affix(); + final String strip = prefix.strip(); + if (affix.isEmpty() && strip.isEmpty() || !word.startsWith(affix) + || word.length() - affix.length() + strip.length() == 0) { return null; } - final String stem = prefix.strip() + word.substring(prefix.affix().length()); + final String stem = strip + word.substring(affix.length()); return prefix.condition().matches(stem) ? stem : null; } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index c6304ab156..82956bd9be 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -375,6 +375,150 @@ void testMalformedFlagValueMessages() { Assertions.assertEquals("unsupported SET encoding: NO-SUCH-ENCODING", e.getMessage()); } + /** + * Verifies that a morphological field is cut off the entry before the flag separator + * is looked for, so a slash inside a morphological field is not mistaken for the + * separator: the entry {@code walk po:verb/noun} registers the word {@code walk} + * with no flags in every flag mode, and its morphology is ignored. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testMorphologicalFieldsAreCutBeforeTheFlagSeparator() throws IOException { + final HunspellDictionary chars = load("SFX G Y 1\nSFX G 0 ing .\n", + "1\nwalk po:verb/noun\n"); + Assertions.assertNotNull(chars.lookup("walk")); + Assertions.assertEquals(0, chars.lookup("walk").get(0).length); + Assertions.assertNull(chars.lookup("walk po:verb")); + + final HunspellDictionary numbers = load("FLAG num\nSFX 1 Y 1\nSFX 1 0 ing .\n", + "1\nwalk po:verb/noun\n"); + Assertions.assertNotNull(numbers.lookup("walk")); + Assertions.assertEquals(0, numbers.lookup("walk").get(0).length); + + // the tabulator is the older morphological field separator + final HunspellDictionary tabbed = load("SFX G Y 1\nSFX G 0 ing .\n", + "1\nwalk\tpo:verb/noun\n"); + Assertions.assertNotNull(tabbed.lookup("walk")); + Assertions.assertEquals(0, tabbed.lookup("walk").get(0).length); + } + + /** + * Verifies that an entry keeps its flags when it carries both a flag run and a + * morphological field holding a slash, in every flag mode. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testFlaggedEntriesKeepTheirFlagsBesideMorphology() throws IOException { + final HunspellDictionary chars = load("SFX A Y 1\nSFX A 0 ing .\n", + "1\nwalk/AB po:verb/noun\n"); + Assertions.assertArrayEquals(new int[] {'A', 'B'}, chars.lookup("walk").get(0)); + Assertions.assertEquals("walk", + new HunspellStemmer(chars).stem("walking").toString()); + + final HunspellDictionary numbers = load("FLAG num\nSFX 1 Y 1\nSFX 1 0 ing .\n", + "1\nwalk/1,2 po:verb/noun\n"); + Assertions.assertArrayEquals(new int[] {1, 2}, numbers.lookup("walk").get(0)); + Assertions.assertEquals("walk", + new HunspellStemmer(numbers).stem("walking").toString()); + } + + /** + * Verifies that a multi-word entry keeps both its spaces and its flags: the word of + * a word-list entry runs up to its morphological fields, not up to its first space. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testMultiWordEntriesKeepTheirSpacesAndFlags() throws IOException { + final HunspellDictionary dictionary = load("FLAG num\nSFX 39 Y 1\nSFX 39 0 s .\n", + "1\nall right/39\n"); + Assertions.assertArrayEquals(new int[] {39}, dictionary.lookup("all right").get(0)); + Assertions.assertNull(dictionary.lookup("all")); + } + + /** + * Verifies that the parser trims word-list entries with the same whitespace + * definition it uses to find their fields: an entry led by a no-break space is + * registered under its real word, both with and without a flag run. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEntriesLedByNoBreakSpaceAreTrimmed() throws IOException { + // \u00A0 is the no-break space, which StringUtil.isWhitespace treats as whitespace + final HunspellDictionary dictionary = load("SFX S Y 1\nSFX S 0 s .\n", + "2\n\u00A0fish\n\u00A0cat/S\n"); + Assertions.assertNotNull(dictionary.lookup("fish")); + Assertions.assertNotNull(dictionary.lookup("cat")); + Assertions.assertNull(dictionary.lookup("")); + Assertions.assertEquals("cat", new HunspellStemmer(dictionary).stem("cats").toString()); + } + + /** + * Verifies that {@code FLAG UTF-8}, which declares single-character flags, is + * accepted and read exactly like the default single-character mode, including a flag + * outside ASCII. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testUtf8FlagModeDeclaresSingleCharacterFlags() throws IOException { + final HunspellStemmer plain = new HunspellStemmer(load(String.join("\n", + "FLAG UTF-8", + "SFX S Y 1", + "SFX S 0 s .", + ""), "1\nwalk/S\n")); + Assertions.assertEquals("walk", plain.stem("walks").toString()); + + // \u00E9 is e with an acute accent, a single-character flag outside ASCII + final HunspellStemmer accented = new HunspellStemmer(load(String.join("\n", + "FLAG UTF-8", + "SFX \u00E9 Y 1", + "SFX \u00E9 0 s .", + ""), "1\nwalk/\u00E9\n")); + Assertions.assertEquals("walk", accented.stem("walks").toString()); + } + + /** + * Verifies that a strip-only rule, whose affix material is empty and which therefore + * only removes stem material, is undone: the suffix rule turns the entry + * {@code bake} into the surface form {@code bak}, and the prefix rule turns + * {@code apple} into {@code pple}. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testStripOnlyAffixRulesAreUndone() throws IOException { + final HunspellStemmer suffixStripping = new HunspellStemmer(load(String.join("\n", + "SFX A Y 1", + "SFX A e 0 e", + ""), "1\nbake/A\n")); + Assertions.assertEquals("bake", suffixStripping.stem("bak").toString()); + + final HunspellStemmer prefixStripping = new HunspellStemmer(load(String.join("\n", + "PFX B Y 1", + "PFX B a 0 a", + ""), "1\napple/B\n")); + Assertions.assertEquals("apple", prefixStripping.stem("pple").toString()); + } + + /** + * Verifies that an entry written with an empty flag run loads and carries no flags in + * every flag mode, rather than failing the load in {@code FLAG num} mode alone. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEmptyFlagRunYieldsNoFlagsInEveryMode() throws IOException { + Assertions.assertEquals(0, load("", "1\nword/\n").lookup("word").get(0).length); + Assertions.assertEquals(0, + load("FLAG long\n", "1\nword/\n").lookup("word").get(0).length); + Assertions.assertEquals(0, + load("FLAG num\n", "1\nword/\n").lookup("word").get(0).length); + } + @Test void testMalformedInputFailsLoud() { Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( From d16f3284c220afeb97b6c39bf655a3484422204b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 01:08:34 -0400 Subject: [PATCH 06/24] OPENNLP-1893: Read flags as code points, tolerate trailing morphology, and stem nothing from nothing Loading the Spanish dictionary of the LibreOffice collection, the same collection this module's README recommends, exposed three gaps against real data. Flags under FLAG UTF-8 are now one code point each instead of one UTF-16 unit, since that dictionary names prefix rules with supplementary characters that would otherwise split into two flags and abort the load; a variation selector after a flag character selects presentation, not identity, and is dropped, which the same file also relies on. A numeric or long flag run ends at the first space or tabulator, the separators the word-list format defines, so trailing morphological text without a tag no longer aborts the load; the morphology cut itself now splits on exactly those two separators, the set the reference implementation's hashmgr.cxx uses, which the javadoc previously claimed while scanning wider whitespace. Stemming the empty word answers the empty word instead of letting a strip-only rule conjure a stem from nothing. All four downloaded dictionaries of the collection, English, Spanish, Hungarian, and German, now load and stem; new tests pin the escaped slash, the multi-word entry with trailing tags, and each corrected behavior. --- .../stemmer/hunspell/HunspellDictionary.java | 60 ++++++-- .../stemmer/hunspell/HunspellStemmer.java | 5 + .../stemmer/hunspell/HunspellStemmerTest.java | 129 ++++++++++++++++++ 3 files changed, 184 insertions(+), 10 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 6dbb001ebe..14177461a4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -25,6 +25,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -374,7 +375,17 @@ private static Map> parseWordList(String content, final int slash = unescapedSlash(entry); if (slash >= 0) { word = entry.substring(0, slash); - flags = parseFlags(entry.substring(slash + 1), flagMode, i + 1); + String flagRun = entry.substring(slash + 1); + // The flag run ends at the first space or tabulator, the separators the + // word-list format defines; whatever follows is a morphological field even + // when it carries no two-letter tag, which hunspell tolerates and so do we. + for (int c = 0; c < flagRun.length(); c++) { + if (isFieldSeparator(flagRun.charAt(c))) { + flagRun = flagRun.substring(0, c); + break; + } + } + flags = parseFlags(flagRun, flagMode, i + 1); } entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) .add(flags); @@ -421,9 +432,12 @@ private static int unescapedSlash(String line) { * Finds where the trailing morphological fields of a word-list entry begin, which * terminates the word and its flag run. A morphological field is either introduced by * a tabulator, the older separator, or written as a two-letter tag followed by - * {@code :} and preceded by whitespace, such as {@code po:verb}. Whitespace that is - * not followed by such a tag belongs to the word, because a word-list entry may name - * several words. + * {@code :} and preceded by a separator, such as {@code po:verb}. A separator that + * is not followed by such a tag belongs to the word, because a word-list entry may + * name several words. The separators are the space and the tabulator, exactly the + * two characters the reference implementation's {@code hashmgr.cxx} splits on; they + * are format delimiters of the word-list grammar, not a whitespace judgment, so + * wider whitespace such as a no-break space stays part of the word by design. * * @param line The trimmed word-list line to scan. * @return The index at which the morphological fields begin, or {@code -1} if the @@ -432,9 +446,9 @@ private static int unescapedSlash(String line) { private static int morphologyIndex(String line) { int cut = -1; for (int i = 4; i < line.length(); i++) { - if (line.charAt(i) == ':' && StringUtil.isWhitespace(line.charAt(i - 3))) { + if (line.charAt(i) == ':' && isFieldSeparator(line.charAt(i - 3))) { int fieldStart = i - 3; - while (fieldStart > 0 && StringUtil.isWhitespace(line.charAt(fieldStart - 1))) { + while (fieldStart > 0 && isFieldSeparator(line.charAt(fieldStart - 1))) { fieldStart--; } // a tag with no word in front of it is not a morphological field @@ -449,6 +463,18 @@ private static int morphologyIndex(String line) { return cut; } + /** + * Checks one character against the word-list format's field separators, space and + * tabulator, the exact set the reference implementation splits morphological fields + * on. + * + * @param c The character to test. + * @return {@code true} if {@code c} separates fields in the word-list format. + */ + private static boolean isFieldSeparator(char c) { + return c == ' ' || c == '\t'; + } + /** * Removes leading and trailing whitespace, using the whitespace definition the rest * of the parser scans with. @@ -509,11 +535,25 @@ private static int[] parseFlags(String text, FlagMode mode, int lineNumber) return flags; } default: { - final int[] flags = new int[text.length()]; - for (int i = 0; i < flags.length; i++) { - flags[i] = text.charAt(i); + // One flag per code point: published dictionaries, the Spanish one of the + // LibreOffice collection among them, name affix rules with supplementary + // characters under FLAG UTF-8, and reading per UTF-16 unit would split such + // a flag into two and reject the rule header as carrying two flags. A + // variation selector after a flag character selects its presentation, the + // emoji telephone against the text telephone, and is no flag of its own; the + // same collection writes such selectors, so they are dropped from flag + // identity. + final int[] buffer = new int[text.codePointCount(0, text.length())]; + int f = 0; + for (int i = 0; i < text.length(); ) { + final int codePoint = text.codePointAt(i); + i += Character.charCount(codePoint); + if (codePoint >= 0xFE00 && codePoint <= 0xFE0F) { + continue; + } + buffer[f++] = codePoint; } - return flags; + return f == buffer.length ? buffer : Arrays.copyOf(buffer, f); } } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index eb55bb6d8e..e933f44881 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -72,6 +72,11 @@ public List stemAll(CharSequence word) { throw new IllegalArgumentException("word must not be null"); } final String surface = word.toString(); + if (surface.isEmpty()) { + // a zero-length word has no morphology; without this guard a strip-only rule + // could restore its strip string onto nothing and answer a non-empty stem + return List.of(surface); + } final Set analyses = new LinkedHashSet<>(); for (final String variant : variants(surface)) { analyze(variant, analyses); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 82956bd9be..b116ec6072 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -539,4 +540,132 @@ void testMalformedInputFailsLoud() { () -> new HunspellStemmerFactory(null)); Assertions.assertThrows(IllegalArgumentException.class, () -> stemmer.stemAll(null)); } + + /** + * Verifies hunspell's tolerance for trailing text after a numeric or long flag run: + * the flag run ends at the first space, and whatever follows is a morphological + * field even without a two-letter tag, so such an entry loads instead of aborting + * the whole dictionary. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testTrailingTextAfterNumericFlagRunIsMorphologyNotAnError() throws IOException { + final HunspellDictionary numbers = load("FLAG num\n", + "2\nwalk/39 blah\nrun/7,9 xyz abc\n"); + Assertions.assertNotNull(numbers.lookup("walk")); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("walk"), 39)); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("run"), 7)); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("run"), 9)); + + final HunspellDictionary longs = load("FLAG long\n", "1\nwalk/AB cd\n"); + Assertions.assertTrue(HunspellDictionary.hasFlag(longs.lookup("walk"), + ('A' << 16) | 'B')); + } + + /** + * Verifies that stemming the empty word answers the empty word: a zero-length + * surface has no morphology, and a strip-only rule must not restore its strip + * string onto nothing and answer a non-empty stem. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEmptyWordStemsToItself() throws IOException { + final HunspellStemmer stripOnly = new HunspellStemmer(load( + "PFX P Y 1\nPFX P xy 0 .\n", + "1\nxy/P\n")); + Assertions.assertEquals("", stripOnly.stem("").toString()); + Assertions.assertEquals(List.of(""), stripOnly.stemAll("")); + } + + /** + * Verifies the escaped-slash feature: {@code \/} belongs to the word, so an entry + * naming a slashed term keeps its slash while the first unescaped slash still + * separates the flag run. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testEscapedSlashBelongsToTheWord() throws IOException { + final HunspellDictionary slashed = load("FLAG num\n", + "2\nTCP\\/IP/39\nAC\\/DC\n"); + Assertions.assertNotNull(slashed.lookup("TCP/IP")); + Assertions.assertTrue(HunspellDictionary.hasFlag(slashed.lookup("TCP/IP"), 39)); + Assertions.assertNotNull(slashed.lookup("AC/DC")); + Assertions.assertNull(slashed.lookup("TCP")); + } + + /** + * Verifies the sharpest combination of the morphology cut: an entry that is both a + * multi-word term and carries trailing tag morphology keeps the whole multi-word + * surface and its flags, and the tags stay out of the word. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testMultiWordEntryWithTrailingTagMorphology() throws IOException { + final HunspellDictionary phrases = load("FLAG num\n", + "1\nall right/39 po:phrase st:allright\n"); + Assertions.assertNotNull(phrases.lookup("all right")); + Assertions.assertTrue(HunspellDictionary.hasFlag(phrases.lookup("all right"), 39)); + Assertions.assertNull(phrases.lookup("all right po:phrase st:allright")); + } + + /** + * Pins FLAG UTF-8 for a supplementary flag character: a flag is one code point, so + * a character above U+FFFF is one flag carrying its code point value, never two + * surrogate-unit flags. The Spanish dictionary of the LibreOffice collection names + * affix rules with such characters, so an affix keyed by a supplementary flag must + * connect to the entries that carry it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testSupplementaryFlagCharacterIsOneCodePointFlag() throws IOException { + // U+1F600 as a flag, written as its surrogate pair + final HunspellDictionary emoji = load("FLAG UTF-8\n", + "1\nwalk/\uD83D\uDE00\n"); + Assertions.assertTrue(HunspellDictionary.hasFlag(emoji.lookup("walk"), 0x1F600)); + Assertions.assertFalse(HunspellDictionary.hasFlag(emoji.lookup("walk"), 0xD83D)); + + final HunspellStemmer stemmer = new HunspellStemmer(load( + "FLAG UTF-8\nSFX \uD83D\uDE00 Y 1\nSFX \uD83D\uDE00 0 s .\n", + "1\nwalk/\uD83D\uDE00\n")); + Assertions.assertEquals("walk", stemmer.stem("walks").toString()); + } + + /** + * Pins the variation-selector rule the Spanish dictionary of the LibreOffice + * collection relies on: a variation selector after a flag character selects its + * presentation and is no flag of its own, so an affix rule named with the emoji + * form of a character connects to entries flagged with either spelling. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testVariationSelectorIsDroppedFromFlagIdentity() throws IOException { + // U+260E BLACK TELEPHONE followed by U+FE0F VARIATION SELECTOR-16, the exact + // shape of a prefix flag in the published es_ES affix file + final HunspellStemmer stemmer = new HunspellStemmer(load( + "FLAG UTF-8\nPFX \u260E\uFE0F Y 1\nPFX \u260E\uFE0F 0 tele .\n", + "1\nfono/\u260E\n")); + Assertions.assertEquals("fono", stemmer.stem("telefono").toString()); + } + + /** + * Pins the documented rejection of rules that neither add nor remove material: a + * suffix rule with strip {@code 0} and affix {@code 0} loads without error and + * never fires, so stemming a flagged dictionary word answers that word exactly + * once. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testRuleThatNeitherAddsNorRemovesLoadsAndNeverFires() throws IOException { + final HunspellStemmer identity = new HunspellStemmer(load( + "SFX X Y 1\nSFX X 0 0 .\n", + "1\nwalk/X\n")); + Assertions.assertEquals(List.of("walk"), identity.stemAll("walk")); + } } From 98320f530cf3b096cd415602f2f13351752bfe6b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 06:59:10 -0400 Subject: [PATCH 07/24] OPENNLP-1893: Resolve numeric dictionary flags through the AF alias table The published Hungarian dictionary flags all of its entries as numeric references into an AF alias table, so without alias support every entry loaded flagless and stemming answered the surface form unchanged. The affix parser now reads the AF table, the first line as the declared count and every further line as one flag run with trailing comments discarded, and a purely numeric flag field in the word list resolves as a 1-based reference into it, failing loud with the line and table size when the reference is out of range. Without an AF table numeric fields keep their FLAG num meaning. The Hungarian dictionary of the LibreOffice collection now stems inflected forms; remaining gaps there are compound territory, which is tracked separately. --- .../stemmer/hunspell/HunspellDictionary.java | 47 ++++++++++++--- .../stemmer/hunspell/HunspellStemmerTest.java | 57 +++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 14177461a4..75d47140db 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -132,8 +132,9 @@ public static HunspellDictionary load(InputStream affixStream, final byte[] affixBytes = readAll(affixStream); final Charset charset = declaredCharset(affixBytes); final AffixFile affix = parseAffix(new String(affixBytes, charset)); - final Map> entries = - parseWordList(new String(readAll(dictionaryStream), charset), affix.flagMode); + final Map> entries = parseWordList( + new String(readAll(dictionaryStream), charset), affix.flagMode, + affix.flagAliases); return new HunspellDictionary(entries, List.copyOf(affix.prefixes), List.copyOf(affix.suffixes)); } @@ -236,14 +237,17 @@ private enum FlagMode { private static final class AffixFile { private final List prefixes = new ArrayList<>(); private final List suffixes = new ArrayList<>(); + private final List flagAliases = new ArrayList<>(); + private boolean aliasHeaderSeen; private FlagMode flagMode = FlagMode.CHAR; } /** - * Parses the affix file: the {@code FLAG} declaration and the {@code PFX} and - * {@code SFX} blocks. Directives outside the supported set (compounding, - * conversion tables, suggestion options, ...) are skipped, so their rules never - * fire and unsupported analyses are missed rather than invented. + * Parses the affix file: the {@code FLAG} declaration, the {@code AF} flag alias + * table, and the {@code PFX} and {@code SFX} blocks. Directives outside the + * supported set (compounding, conversion tables, suggestion options, ...) are + * skipped, so their rules never fire and unsupported analyses are missed rather + * than invented. * * @param content The decoded affix file content. * @return The parsed rules and flag mode. Never {@code null}. @@ -273,6 +277,18 @@ private static AffixFile parseAffix(String content) throws IOException { }; i++; break; + case "AF": + // the first AF line declares the alias count; every further AF line is one + // alias, a flag run whose 1-based position numeric dictionary flags refer to + if (fields.length >= 2) { + if (!result.aliasHeaderSeen) { + result.aliasHeaderSeen = true; + } else { + result.flagAliases.add(parseFlags(fields[1], result.flagMode, i + 1)); + } + } + i++; + break; case "PFX": case "SFX": i = parseAffixBlock(lines, i, fields, result); @@ -352,11 +368,15 @@ private static int parseAffixBlock(String[] lines, int index, String[] header, * * @param content The decoded word-list content. * @param flagMode The flag encoding declared by the affix file. + * @param flagAliases The affix file's {@code AF} alias table, possibly empty. When + * it is not empty, a purely numeric flag field is a 1-based + * reference into it rather than a flag run of its own. * @return The words mapped to the flag sets of their entries. Never {@code null}. - * @throws IOException Thrown if a flag run is malformed. + * @throws IOException Thrown if a flag run is malformed or an alias reference is + * out of range. */ private static Map> parseWordList(String content, - FlagMode flagMode) throws IOException { + FlagMode flagMode, List flagAliases) throws IOException { final String[] lines = splitLines(content); final Map> entries = new HashMap<>(); int start = 0; @@ -385,7 +405,16 @@ private static Map> parseWordList(String content, break; } } - flags = parseFlags(flagRun, flagMode, i + 1); + if (!flagAliases.isEmpty() && isCount(flagRun)) { + final int alias = Integer.parseInt(flagRun); + if (alias < 1 || alias > flagAliases.size()) { + throw new IOException("flag alias " + alias + " at line " + (i + 1) + + " is outside the AF table of " + flagAliases.size() + " aliases"); + } + flags = flagAliases.get(alias - 1); + } else { + flags = parseFlags(flagRun, flagMode, i + 1); + } } entries.computeIfAbsent(word.replace("\\/", "/"), key -> new ArrayList<>(1)) .add(flags); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index b116ec6072..66483c701a 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -668,4 +668,61 @@ void testRuleThatNeitherAddsNorRemovesLoadsAndNeverFires() throws IOException { "1\nwalk/X\n")); Assertions.assertEquals(List.of("walk"), identity.stemAll("walk")); } + + /** + * Verifies the AF flag alias table: the first AF line declares the count, every + * further AF line is one flag run, and a purely numeric flag field in the word + * list is a 1-based reference into that table, the layout the published Hungarian + * dictionary uses for all of its ninety-seven thousand entries. Alias lines may + * carry trailing comments, which the field split already discards. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testNumericDictionaryFlagsResolveThroughTheAliasTable() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + String.join("\n", + "AF 2", + "AF S # 1", + "AF SP # 2", + "SFX S Y 1", + "SFX S 0 s .", + "PFX P Y 1", + "PFX P 0 re .", + ""), + "2\nwalk/1\nplay/2\n")); + Assertions.assertEquals("walk", stemmer.stem("walks").toString()); + Assertions.assertEquals("play", stemmer.stem("plays").toString()); + Assertions.assertEquals("play", stemmer.stem("replay").toString()); + // walk carries alias 1, the suffix-only run, so the prefix must not apply + Assertions.assertEquals("rewalk", stemmer.stem("rewalk").toString()); + } + + /** + * Verifies that an alias reference outside the AF table fails loud with the line + * and the table size, instead of silently flagging the entry with nothing. + * + * @throws IOException Thrown if the affix fixture fails to load. + */ + @Test + void testAliasReferenceOutsideTheTableFailsLoud() { + final IOException e = Assertions.assertThrows(IOException.class, () -> load( + "AF 1\nAF S # 1\nSFX S Y 1\nSFX S 0 s .\n", + "1\nwalk/2\n")); + Assertions.assertEquals( + "flag alias 2 at line 2 is outside the AF table of 1 aliases", + e.getMessage()); + } + + /** + * Verifies that numeric flag fields stay ordinary flags when no AF table exists: + * under FLAG num a digit run is a flag value, not an alias reference. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testNumericFlagsWithoutAliasTableStayFlags() throws IOException { + final HunspellDictionary numbers = load("FLAG num\n", "1\nwalk/39\n"); + Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("walk"), 39)); + } } From e80f4d1937d9aeb1119e4ae72ec9049963ca2e34 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:03:53 -0400 Subject: [PATCH 08/24] OPENNLP-1893: Walk only the affix rules that can apply, bucketed by their boundary character Undoing a suffix requires the word to end with the rule's affix material, so only rules whose material ends in the word's last character can ever apply, and likewise for prefixes and the first character. The dictionary now buckets its rules by that boundary character at load, and every scan in the stem path, including the twofold and cross-product inner scans, walks the one bucket plus the strip-only rules instead of the whole inventory. Measured on the LibreOffice dictionaries at 4,000 words each: English 553k to 1,024k words per second, Spanish 9.6k to 28.9k, German 132k to 287k. --- .../stemmer/hunspell/HunspellDictionary.java | 66 +++++++ .../stemmer/hunspell/HunspellStemmer.java | 164 ++++++++++++------ 2 files changed, 181 insertions(+), 49 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 75d47140db..c9e8f8a6e7 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -85,12 +85,44 @@ boolean allowsContinuation(int otherFlag) { private final Map> entries; private final List prefixes; private final List suffixes; + private final Map> suffixesByLast; + private final List suffixesWithoutMaterial; + private final Map> prefixesByFirst; + private final List prefixesWithoutMaterial; private HunspellDictionary(Map> entries, List prefixes, List suffixes) { this.entries = entries; this.prefixes = prefixes; this.suffixes = suffixes; + // Undoing a suffix requires the word to end with the rule's affix material, so + // only rules whose material ends in the word's last character can ever apply; + // the same holds for prefixes and the first character. Bucketing by that + // boundary character turns the per-word rule scan from the whole inventory into + // the one bucket plus the strip-only rules, whose empty material matches + // everywhere. + this.suffixesByLast = new HashMap<>(); + this.suffixesWithoutMaterial = new ArrayList<>(); + for (final Affix suffix : suffixes) { + final String material = suffix.affix(); + if (material.isEmpty()) { + suffixesWithoutMaterial.add(suffix); + } else { + suffixesByLast.computeIfAbsent(material.charAt(material.length() - 1), + key -> new ArrayList<>()).add(suffix); + } + } + this.prefixesByFirst = new HashMap<>(); + this.prefixesWithoutMaterial = new ArrayList<>(); + for (final Affix prefix : prefixes) { + final String material = prefix.affix(); + if (material.isEmpty()) { + prefixesWithoutMaterial.add(prefix); + } else { + prefixesByFirst.computeIfAbsent(material.charAt(0), + key -> new ArrayList<>()).add(prefix); + } + } } /** @@ -159,6 +191,40 @@ List suffixes() { return suffixes; } + private static final List NO_AFFIXES = List.of(); + + /** + * The suffix rules whose affix material ends in the given character, which are the + * only material-bearing rules that can be undone from a word ending in it. + * + * @param last The word's last character. + * @return The bucket, possibly empty. Never {@code null}. + */ + List suffixesEndingWith(char last) { + return suffixesByLast.getOrDefault(last, NO_AFFIXES); + } + + /** @return The strip-only suffix rules, applicable to any word. Never {@code null}. */ + List suffixesWithoutMaterial() { + return suffixesWithoutMaterial; + } + + /** + * The prefix rules whose affix material starts with the given character, which are + * the only material-bearing rules that can be undone from a word starting with it. + * + * @param first The word's first character. + * @return The bucket, possibly empty. Never {@code null}. + */ + List prefixesStartingWith(char first) { + return prefixesByFirst.getOrDefault(first, NO_AFFIXES); + } + + /** @return The strip-only prefix rules, applicable to any word. Never {@code null}. */ + List prefixesWithoutMaterial() { + return prefixesWithoutMaterial; + } + /** * Checks whether any of a word's flag sets carries a flag. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index e933f44881..6e0d369c06 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -114,55 +114,121 @@ private void analyze(String word, Set analyses) { if (dictionary.lookup(word) != null) { analyses.add(word); } - for (final Affix suffix : dictionary.suffixes()) { - final String stem = removeSuffix(word, suffix); - if (stem == null) { - continue; - } - final List flagSets = dictionary.lookup(stem); - if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { - analyses.add(stem); - } - for (final Affix inner : dictionary.suffixes()) { - if (!inner.allowsContinuation(suffix.flag())) { - continue; - } - final String doubleStem = removeSuffix(stem, inner); - if (doubleStem == null) { - continue; - } - final List innerFlags = dictionary.lookup(doubleStem); - if (innerFlags != null && HunspellDictionary.hasFlag(innerFlags, inner.flag())) { - analyses.add(doubleStem); - } - } - } - for (final Affix prefix : dictionary.prefixes()) { - final String stem = removePrefix(word, prefix); - if (stem == null) { - continue; - } - final List flagSets = dictionary.lookup(stem); - if (flagSets != null && HunspellDictionary.hasFlag(flagSets, prefix.flag())) { - analyses.add(stem); - } - if (!prefix.crossProduct()) { - continue; - } - for (final Affix suffix : dictionary.suffixes()) { - if (!suffix.crossProduct()) { - continue; - } - final String doubleStem = removeSuffix(stem, suffix); - if (doubleStem == null) { - continue; - } - final List both = dictionary.lookup(doubleStem); - if (both != null && HunspellDictionary.hasFlag(both, prefix.flag()) - && HunspellDictionary.hasFlag(both, suffix.flag())) { - analyses.add(doubleStem); - } - } + // Only rules whose affix material ends in the word's last character can be + // undone from it, so each scan walks that bucket plus the strip-only rules + // instead of the whole inventory. + for (final Affix suffix : dictionary.suffixesEndingWith(word.charAt(word.length() - 1))) { + undoSuffix(word, suffix, analyses); + } + for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { + undoSuffix(word, suffix, analyses); + } + for (final Affix prefix : dictionary.prefixesStartingWith(word.charAt(0))) { + undoPrefix(word, prefix, analyses); + } + for (final Affix prefix : dictionary.prefixesWithoutMaterial()) { + undoPrefix(word, prefix, analyses); + } + } + + /** + * Undoes one suffix rule and, through continuation classes, one further suffix on + * the intermediate stem, adding every dictionary-confirmed analysis. + * + * @param word The case variant under analysis. + * @param suffix The suffix rule to undo. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoSuffix(String word, Affix suffix, Set analyses) { + final String stem = removeSuffix(word, suffix); + if (stem == null) { + return; + } + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { + analyses.add(stem); + } + for (final Affix inner : dictionary.suffixesEndingWith(stem.charAt(stem.length() - 1))) { + undoInnerSuffix(stem, suffix, inner, analyses); + } + for (final Affix inner : dictionary.suffixesWithoutMaterial()) { + undoInnerSuffix(stem, suffix, inner, analyses); + } + } + + /** + * Undoes the second suffix of a twofold removal when the inner rule's continuation + * classes allow it after the outer one. + * + * @param stem The intermediate stem after the outer removal. + * @param outer The already-undone outer suffix rule. + * @param inner The candidate inner suffix rule. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoInnerSuffix(String stem, Affix outer, Affix inner, + Set analyses) { + if (!inner.allowsContinuation(outer.flag())) { + return; + } + final String doubleStem = removeSuffix(stem, inner); + if (doubleStem == null) { + return; + } + final List innerFlags = dictionary.lookup(doubleStem); + if (innerFlags != null && HunspellDictionary.hasFlag(innerFlags, inner.flag())) { + analyses.add(doubleStem); + } + } + + /** + * Undoes one prefix rule and, for cross-product rules, one further suffix on the + * intermediate stem, adding every dictionary-confirmed analysis. + * + * @param word The case variant under analysis. + * @param prefix The prefix rule to undo. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoPrefix(String word, Affix prefix, Set analyses) { + final String stem = removePrefix(word, prefix); + if (stem == null) { + return; + } + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && HunspellDictionary.hasFlag(flagSets, prefix.flag())) { + analyses.add(stem); + } + if (!prefix.crossProduct()) { + return; + } + for (final Affix suffix : dictionary.suffixesEndingWith(stem.charAt(stem.length() - 1))) { + undoCrossProductSuffix(stem, prefix, suffix, analyses); + } + for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { + undoCrossProductSuffix(stem, prefix, suffix, analyses); + } + } + + /** + * Undoes the suffix half of a cross-product removal when both rules opted in. + * + * @param stem The intermediate stem after the prefix removal. + * @param prefix The already-undone prefix rule. + * @param suffix The candidate suffix rule. + * @param analyses The mutable, insertion-ordered set collecting the stems found. + */ + private void undoCrossProductSuffix(String stem, Affix prefix, Affix suffix, + Set analyses) { + if (!suffix.crossProduct()) { + return; + } + final String doubleStem = removeSuffix(stem, suffix); + if (doubleStem == null) { + return; + } + final List both = dictionary.lookup(doubleStem); + if (both != null && HunspellDictionary.hasFlag(both, prefix.flag()) + && HunspellDictionary.hasFlag(both, suffix.flag())) { + analyses.add(doubleStem); } } From 2f0eab31eee44b2c492396b32ef61723ed037beb Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:23:42 -0400 Subject: [PATCH 09/24] OPENNLP-1893: Decompose unanalyzed words into two flagged compound parts When the affix analysis finds nothing and the affix file declares compounding, a word now splits into two listed parts that the COMPOUNDFLAG or the positional COMPOUNDBEGIN and COMPOUNDEND flags allow in their positions, honoring COMPOUNDMIN, with the parts reported left to right. Affix analyses keep precedence, listed words never decompose, and unflagged parts block a split. Against the published Hungarian dictionary the unlisted kutyahaz decomposes into its two nouns while listed compounds and inflected forms keep their regular analyses. Longer chains, syllable rules, and the compound-only flags stay unimplemented and simply leave such words unanalyzed. --- .../stemmer/hunspell/HunspellDictionary.java | 77 ++++++++++++++++++- .../stemmer/hunspell/HunspellStemmer.java | 34 ++++++++ .../stemmer/hunspell/HunspellStemmerTest.java | 50 ++++++++++++ 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index c9e8f8a6e7..0f51d0e24d 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -89,9 +89,18 @@ boolean allowsContinuation(int otherFlag) { private final List suffixesWithoutMaterial; private final Map> prefixesByFirst; private final List prefixesWithoutMaterial; + private final int compoundFlag; + private final int compoundBegin; + private final int compoundEnd; + private final int compoundMin; private HunspellDictionary(Map> entries, List prefixes, - List suffixes) { + List suffixes, int compoundFlag, int compoundBegin, int compoundEnd, + int compoundMin) { + this.compoundFlag = compoundFlag; + this.compoundBegin = compoundBegin; + this.compoundEnd = compoundEnd; + this.compoundMin = compoundMin; this.entries = entries; this.prefixes = prefixes; this.suffixes = suffixes; @@ -168,7 +177,8 @@ public static HunspellDictionary load(InputStream affixStream, new String(readAll(dictionaryStream), charset), affix.flagMode, affix.flagAliases); return new HunspellDictionary(entries, List.copyOf(affix.prefixes), - List.copyOf(affix.suffixes)); + List.copyOf(affix.suffixes), affix.compoundFlag, affix.compoundBegin, + affix.compoundEnd, affix.compoundMin); } /** @@ -225,6 +235,40 @@ List prefixesWithoutMaterial() { return prefixesWithoutMaterial; } + /** @return Whether the affix file declares any compounding flag at all. */ + boolean compoundsDeclared() { + return compoundFlag != 0 || compoundBegin != 0 || compoundEnd != 0; + } + + /** @return The smallest length a compound part may have; at least {@code 1}. */ + int compoundMin() { + return compoundMin; + } + + /** + * Checks whether a listed word may open a compound: it carries the general + * compounding flag or the dedicated begin flag. + * + * @param flagSets The word's flag sets from {@link #lookup(String)}. + * @return {@code true} if the word may stand first in a compound. + */ + boolean mayBeginCompound(List flagSets) { + return (compoundFlag != 0 && hasFlag(flagSets, compoundFlag)) + || (compoundBegin != 0 && hasFlag(flagSets, compoundBegin)); + } + + /** + * Checks whether a listed word may close a compound: it carries the general + * compounding flag or the dedicated end flag. + * + * @param flagSets The word's flag sets from {@link #lookup(String)}. + * @return {@code true} if the word may stand last in a compound. + */ + boolean mayEndCompound(List flagSets) { + return (compoundFlag != 0 && hasFlag(flagSets, compoundFlag)) + || (compoundEnd != 0 && hasFlag(flagSets, compoundEnd)); + } + /** * Checks whether any of a word's flag sets carries a flag. * @@ -306,6 +350,10 @@ private static final class AffixFile { private final List flagAliases = new ArrayList<>(); private boolean aliasHeaderSeen; private FlagMode flagMode = FlagMode.CHAR; + private int compoundFlag; + private int compoundBegin; + private int compoundEnd; + private int compoundMin = 3; } /** @@ -343,6 +391,31 @@ private static AffixFile parseAffix(String content) throws IOException { }; i++; break; + case "COMPOUNDFLAG": + case "COMPOUNDBEGIN": + case "COMPOUNDEND": + if (fields.length < 2) { + throw new IOException(fields[0] + " line without a flag at line " + (i + 1)); + } + final int compound = parseFlag(fields[1], result.flagMode, i + 1); + switch (fields[0]) { + case "COMPOUNDFLAG" -> result.compoundFlag = compound; + case "COMPOUNDBEGIN" -> result.compoundBegin = compound; + default -> result.compoundEnd = compound; + } + i++; + break; + case "COMPOUNDMIN": + if (fields.length < 2) { + throw new IOException("COMPOUNDMIN line without a value at line " + (i + 1)); + } + try { + result.compoundMin = Math.max(1, Integer.parseInt(fields[1])); + } catch (NumberFormatException e) { + throw new IOException("malformed COMPOUNDMIN at line " + (i + 1), e); + } + i++; + break; case "AF": // the first AF line declares the alias count; every further AF line is one // alias, a flag run whose 1-based position numeric dictionary flags refer to diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 6e0d369c06..6ac1a48aa0 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -81,6 +81,11 @@ public List stemAll(CharSequence word) { for (final String variant : variants(surface)) { analyze(variant, analyses); } + if (analyses.isEmpty() && dictionary.compoundsDeclared()) { + for (final String variant : variants(surface)) { + decompose(variant, analyses); + } + } if (analyses.isEmpty()) { return List.of(surface); } @@ -131,6 +136,35 @@ private void analyze(String word, Set analyses) { } } + /** + * Decomposes a word into two listed compound parts when the affix analysis found + * nothing: at every split point that leaves both sides at least the declared + * minimum length, the left side must be listed and allowed to open a compound and + * the right side listed and allowed to close one. The parts of the first splitting + * that succeeds are reported left to right, so the head-most material comes last, + * and further splittings add any parts not already reported. + * + * @param word The case variant to decompose. + * @param analyses The mutable, insertion-ordered set collecting the parts. + */ + private void decompose(String word, Set analyses) { + final int min = dictionary.compoundMin(); + for (int split = min; split <= word.length() - min; split++) { + final String left = word.substring(0, split); + final List leftFlags = dictionary.lookup(left); + if (leftFlags == null || !dictionary.mayBeginCompound(leftFlags)) { + continue; + } + final String right = word.substring(split); + final List rightFlags = dictionary.lookup(right); + if (rightFlags == null || !dictionary.mayEndCompound(rightFlags)) { + continue; + } + analyses.add(left); + analyses.add(right); + } + } + /** * Undoes one suffix rule and, through continuation classes, one further suffix on * the intermediate stem, adding every dictionary-confirmed analysis. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 66483c701a..9ffd1289b2 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -725,4 +725,54 @@ void testNumericFlagsWithoutAliasTableStayFlags() throws IOException { final HunspellDictionary numbers = load("FLAG num\n", "1\nwalk/39\n"); Assertions.assertTrue(HunspellDictionary.hasFlag(numbers.lookup("walk"), 39)); } + + /** + * Verifies two-part compound decomposition under the general compounding flag: a + * word the affix analysis cannot explain splits into two listed parts that both + * carry the flag, reported left to right, while a part without the flag blocks the + * split and the word stays unanalyzed. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundFlagDecomposesUnanalyzedWords() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\n", + "3\ndog/Z\nhouse/Z\ncat\n")); + Assertions.assertEquals(List.of("dog", "house"), stemmer.stemAll("doghouse")); + // cat is listed without the compounding flag, so no split may use it + Assertions.assertEquals(List.of("cathouse"), stemmer.stemAll("cathouse")); + // a listed word never decomposes; it is its own analysis + Assertions.assertEquals(List.of("dog"), stemmer.stemAll("dog")); + } + + /** + * Verifies the positional compound flags: the begin flag only opens and the end + * flag only closes, so the parts compose in one order and refuse the other. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundBeginAndEndFlagsArePositional() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDBEGIN B\nCOMPOUNDEND E\nCOMPOUNDMIN 3\n", + "2\ndog/B\nhouse/E\n")); + Assertions.assertEquals(List.of("dog", "house"), stemmer.stemAll("doghouse")); + Assertions.assertEquals(List.of("housedog"), stemmer.stemAll("housedog")); + } + + /** + * Verifies the minimum part length: a split leaving a side shorter than + * COMPOUNDMIN is never taken, although both sides are listed and flagged. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundMinBoundsThePartLength() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 4\n", + "2\ndog/Z\nhouse/Z\n")); + // the left side would be three characters, below the declared minimum of four + Assertions.assertEquals(List.of("doghouse"), stemmer.stemAll("doghouse")); + } } From 7b7d2c4583e657e79b15781d2d08b3ce07bcedab Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 00:43:29 -0400 Subject: [PATCH 10/24] OPENNLP-1893: Honor the blocking flags, circumfixes, and compound positioning A NEEDAFFIX (or PSEUDOROOT) entry is a virtual stem that exists only to be affixed, an ONLYINCOMPOUND entry appears only inside compounds, and a FORBIDDENWORD entry is listed to be blocked; none of them is a standalone analysis anymore, per homonym flag set, and an affix carrying NEEDAFFIX among its continuation classes yields no single-removal analysis while its twofold and cross-product removals stand, the other affix being exactly the further one required. A cross-product now also requires both removed affixes' flags in the same homonym's flag set, and CIRCUMFIX binds marked prefix and suffix halves to one another, so neither half analyzes alone and a marked half never combines with an unmarked affix. Decomposition grows from two verbatim parts to the compound machinery the published German dictionary actually uses: any number of parts under the positional COMPOUNDBEGIN/COMPOUNDMIDDLE/COMPOUNDEND flags and COMPOUNDWORDMAX, parts standing on an entry plus one affix with COMPOUNDPERMITFLAG required at internal boundaries and COMPOUNDFORBIDFLAG barring marked forms, zero and dash linking suffixes included, an uppercased retry for capitalized entries spelled lowercase inside a compound, and the CHECKCOMPOUNDDUP, CHECKCOMPOUNDCASE, and CHECKCOMPOUNDTRIPLE junction guards, case judged against the original surface. A listed forbidden word never decomposes, and a fixed part-licensing budget keeps adversarial input bounded, missing analyses rather than stalling. Abbildungsverzeichnis, Haustuer, and Kinderzimmer now decompose against de_DE_frami at 137k words/s single-threaded. An opt-in test class checks everyday morphology against downloaded dictionaries under -Dopennlp.hunspell.dict.dir; nothing is bundled. --- .../dev/README-hunspell-dictionaries.md | 11 +- .../stemmer/hunspell/HunspellDictionary.java | 395 ++++++++++++++++-- .../stemmer/hunspell/HunspellStemmer.java | 356 ++++++++++++++-- .../hunspell/HunspellRealDictionaryTest.java | 99 +++++ .../stemmer/hunspell/HunspellStemmerTest.java | 379 +++++++++++++++++ 5 files changed, 1159 insertions(+), 81 deletions(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java diff --git a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md index 31edf4fe0b..457fd01eeb 100644 --- a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md +++ b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md @@ -50,6 +50,15 @@ What `stem` evaluates to is decided by the dictionary you loaded, and this proje The dictionary is immutable and safe to share between threads; the factory hands out a fresh stemmer per call, so each thread takes its own from `newStemmer()`. A dictionary that declares a non-UTF-8 encoding through the `SET` directive in its `.aff` file is decoded accordingly; nothing needs converting beforehand. +## Testing against real dictionaries + +The in-tree tests run against project-authored fixtures only. An opt-in test class, `HunspellRealDictionaryTest`, additionally checks everyday morphology against published dictionaries when pointed at a directory of `.aff`/`.dic` pairs (each test skips when its pair is absent): + +``` +./mvnw test -pl opennlp-core/opennlp-runtime -Dtest=HunspellRealDictionaryTest \ + -Dopennlp.hunspell.dict.dir=/tmp/hunspell-dicts +``` + ## What the engine supports -Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, and the `SET` encoding declaration. Compounding and conversion tables are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, the `AF` flag alias table, the `SET` encoding declaration, compound decomposition under `COMPOUNDFLAG`, the positional `COMPOUNDBEGIN`/`COMPOUNDMIDDLE`/`COMPOUNDEND` flags, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, and the `CHECKCOMPOUNDDUP`/`CHECKCOMPOUNDCASE`/`CHECKCOMPOUNDTRIPLE` declarations (compound parts stand on their entries alone or on an entry plus one affix, the zero and dash suffixes dictionaries position linking forms with included), the blocking flags `NEEDAFFIX` (alias `PSEUDOROOT`), `ONLYINCOMPOUND`, and `FORBIDDENWORD`, which keep virtual stems, compound-only parts, and forbidden words out of the reported analyses, and `CIRCUMFIX`, which binds marked prefix and suffix halves to one another as in the German `ge...t` participle. Conversion tables and the remaining compound machinery are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 0f51d0e24d..7197185ffb 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -43,10 +43,21 @@ * character-class conditions, and cross-product combination of one prefix with one * suffix; twofold suffixes through the continuation classes on suffix rules; * {@code FLAG} modes {@code char} (default), {@code UTF-8}, {@code long}, and - * {@code num}; the {@code SET} encoding declaration. Compounding and conversion tables - * are not - * interpreted in this version; rules using them simply do not fire, so unsupported - * analyses are missed rather than invented.

+ * {@code num}; the {@code AF} flag alias table; the {@code SET} encoding declaration; + * compound decomposition under {@code COMPOUNDFLAG}, the positional + * {@code COMPOUNDBEGIN}/{@code COMPOUNDMIDDLE}/{@code COMPOUNDEND} flags, + * {@code COMPOUNDMIN}, {@code COMPOUNDWORDMAX}, {@code COMPOUNDPERMITFLAG}, + * {@code COMPOUNDFORBIDFLAG}, and the {@code CHECKCOMPOUNDDUP}, + * {@code CHECKCOMPOUNDCASE}, and {@code CHECKCOMPOUNDTRIPLE} declarations, with + * compound parts standing on their entries alone or on an entry plus one affix; the + * blocking flags + * {@code NEEDAFFIX} (with its historical alias {@code PSEUDOROOT}), + * {@code ONLYINCOMPOUND}, and {@code FORBIDDENWORD}, which suppress analyses the + * dictionary marks as virtual stems, compound-only parts, or forbidden words; and + * {@code CIRCUMFIX}, which binds marked prefix and suffix halves to one another. + * Conversion tables and the remaining compound machinery are not interpreted in this + * version; rules using them simply do not fire, so unsupported analyses are missed + * rather than invented.

* *

Instances are immutable and safe to share between threads.

* @@ -93,17 +104,47 @@ boolean allowsContinuation(int otherFlag) { private final int compoundBegin; private final int compoundEnd; private final int compoundMin; + /** The place a part takes in a compound, deciding which positional flag admits it. */ + enum CompoundPosition { + /** The first part. */ + BEGIN, + /** Any part between the first and the last. */ + MIDDLE, + /** The last part. */ + END + } + + private final int needAffix; + private final int onlyInCompound; + private final int forbiddenWord; + private final int circumfix; + private final int compoundMiddle; + private final int compoundPermit; + private final int compoundForbid; + private final int compoundWordMax; + private final boolean checkCompoundDup; + private final boolean checkCompoundCase; + private final boolean checkCompoundTriple; - private HunspellDictionary(Map> entries, List prefixes, - List suffixes, int compoundFlag, int compoundBegin, int compoundEnd, - int compoundMin) { - this.compoundFlag = compoundFlag; - this.compoundBegin = compoundBegin; - this.compoundEnd = compoundEnd; - this.compoundMin = compoundMin; + private HunspellDictionary(Map> entries, AffixFile affix) { + this.compoundFlag = affix.compoundFlag; + this.compoundBegin = affix.compoundBegin; + this.compoundEnd = affix.compoundEnd; + this.compoundMin = affix.compoundMin; + this.needAffix = affix.needAffix; + this.onlyInCompound = affix.onlyInCompound; + this.forbiddenWord = affix.forbiddenWord; + this.circumfix = affix.circumfix; + this.compoundMiddle = affix.compoundMiddle; + this.compoundPermit = affix.compoundPermit; + this.compoundForbid = affix.compoundForbid; + this.compoundWordMax = affix.compoundWordMax; + this.checkCompoundDup = affix.checkCompoundDup; + this.checkCompoundCase = affix.checkCompoundCase; + this.checkCompoundTriple = affix.checkCompoundTriple; this.entries = entries; - this.prefixes = prefixes; - this.suffixes = suffixes; + this.prefixes = List.copyOf(affix.prefixes); + this.suffixes = List.copyOf(affix.suffixes); // Undoing a suffix requires the word to end with the rule's affix material, so // only rules whose material ends in the word's last character can ever apply; // the same holds for prefixes and the first character. Bucketing by that @@ -176,9 +217,7 @@ public static HunspellDictionary load(InputStream affixStream, final Map> entries = parseWordList( new String(readAll(dictionaryStream), charset), affix.flagMode, affix.flagAliases); - return new HunspellDictionary(entries, List.copyOf(affix.prefixes), - List.copyOf(affix.suffixes), affix.compoundFlag, affix.compoundBegin, - affix.compoundEnd, affix.compoundMin); + return new HunspellDictionary(entries, affix); } /** @@ -237,7 +276,8 @@ List prefixesWithoutMaterial() { /** @return Whether the affix file declares any compounding flag at all. */ boolean compoundsDeclared() { - return compoundFlag != 0 || compoundBegin != 0 || compoundEnd != 0; + return compoundFlag != 0 || compoundBegin != 0 || compoundEnd != 0 + || compoundMiddle != 0; } /** @return The smallest length a compound part may have; at least {@code 1}. */ @@ -245,28 +285,136 @@ int compoundMin() { return compoundMin; } + /** @return The largest number of parts a compound may have; {@code 0} is unbounded. */ + int compoundWordMax() { + return compoundWordMax; + } + + /** @return Whether {@code CHECKCOMPOUNDDUP} forbids a part repeating its neighbor. */ + boolean checkCompoundDup() { + return checkCompoundDup; + } + + /** @return Whether {@code CHECKCOMPOUNDCASE} forbids uppercase at part boundaries. */ + boolean checkCompoundCase() { + return checkCompoundCase; + } + + /** @return Whether {@code CHECKCOMPOUNDTRIPLE} forbids triple letters at boundaries. */ + boolean checkCompoundTriple() { + return checkCompoundTriple; + } + /** - * Checks whether a listed word may open a compound: it carries the general - * compounding flag or the dedicated begin flag. + * The flag admitting a part at a compound position, next to the general + * compounding flag. + * + * @param position The part's place in the compound. + * @return The dedicated positional flag, or {@code 0} when undeclared. + */ + private int positionalFlag(CompoundPosition position) { + return switch (position) { + case BEGIN -> compoundBegin; + case MIDDLE -> compoundMiddle; + case END -> compoundEnd; + }; + } + + /** + * Checks whether a listed word may stand at a compound position: some homonym's + * flag set carries the general compounding flag or the position's dedicated flag + * and is not forbidden. A compound-only or virtual-stem homonym may take the + * position; that is what those flags permit. * * @param flagSets The word's flag sets from {@link #lookup(String)}. - * @return {@code true} if the word may stand first in a compound. + * @param position The part's place in the compound. + * @return {@code true} if the word may stand at the position. + */ + boolean mayStand(List flagSets, CompoundPosition position) { + final int positional = positionalFlag(position); + for (final int[] flags : flagSets) { + if ((contains(flags, compoundFlag) || contains(flags, positional)) + && !contains(flags, forbiddenWord) && !contains(flags, needAffix)) { + return true; + } + } + return false; + } + + /** + * Checks whether some homonym supports an affixed compound part: its flag set + * carries the removed affix's flag, is not forbidden, and either the affix itself + * admits the position or the set carries the compounding or positional flag. + * + * @param flagSets The part stem's flag sets from {@link #lookup(String)}. + * @param affixFlag The removed affix's flag. + * @param position The part's place in the compound. + * @param affixAdmits Whether the affix's continuation classes admit the position, + * from {@link #affixAdmits(Affix, CompoundPosition)}. + * @return {@code true} if some homonym stands affixed at the position. + */ + boolean supportsPart(List flagSets, int affixFlag, CompoundPosition position, + boolean affixAdmits) { + final int positional = positionalFlag(position); + for (final int[] flags : flagSets) { + if (contains(flags, affixFlag) && !contains(flags, forbiddenWord) + && (affixAdmits || contains(flags, compoundFlag) + || contains(flags, positional))) { + return true; + } + } + return false; + } + + /** + * Checks whether an affix admits its derived form at a compound position: its + * continuation classes carry the general compounding flag or the position's + * dedicated flag. Published dictionaries position their linking forms this way, + * through zero or dash suffixes whose continuation classes hold the positional + * flags. + * + * @param affix The affix rule applied to the part. + * @param position The part's place in the compound. + * @return {@code true} if the affixed form may stand at the position. + */ + boolean affixAdmits(Affix affix, CompoundPosition position) { + return (compoundFlag != 0 && affix.allowsContinuation(compoundFlag)) + || (positionalFlag(position) != 0 + && affix.allowsContinuation(positionalFlag(position))); + } + + /** + * Checks whether an affix may sit at a compound-internal boundary: it carries the + * {@code COMPOUNDPERMITFLAG} among its continuation classes. Without the flag a + * suffix fits only the last part and a prefix only the first. + * + * @param affix The affix rule applied to the part. + * @return {@code true} if the affix may face another part. */ - boolean mayBeginCompound(List flagSets) { - return (compoundFlag != 0 && hasFlag(flagSets, compoundFlag)) - || (compoundBegin != 0 && hasFlag(flagSets, compoundBegin)); + boolean permitsInside(Affix affix) { + return compoundPermit != 0 && affix.allowsContinuation(compoundPermit); } /** - * Checks whether a listed word may close a compound: it carries the general - * compounding flag or the dedicated end flag. + * Checks whether an affix bars its derived form from compounds altogether: it + * carries the {@code COMPOUNDFORBIDFLAG} among its continuation classes. + * + * @param affix The affix rule applied to the part. + * @return {@code true} if the affixed form may not join a compound. + */ + boolean forbidsInCompound(Affix affix) { + return compoundForbid != 0 && affix.allowsContinuation(compoundForbid); + } + + /** + * Checks whether any of a word's flag sets is forbidden, which a dictionary uses + * to block one specific ill-formed compound while its parts stay productive. * * @param flagSets The word's flag sets from {@link #lookup(String)}. - * @return {@code true} if the word may stand last in a compound. + * @return {@code true} if some homonym carries the forbidden-word flag. */ - boolean mayEndCompound(List flagSets) { - return (compoundFlag != 0 && hasFlag(flagSets, compoundFlag)) - || (compoundEnd != 0 && hasFlag(flagSets, compoundEnd)); + boolean anyForbidden(List flagSets) { + return hasFlag(flagSets, forbiddenWord); } /** @@ -278,15 +426,130 @@ boolean mayEndCompound(List flagSets) { */ static boolean hasFlag(List flagSets, int flag) { for (final int[] flags : flagSets) { - for (final int candidate : flags) { - if (candidate == flag) { - return true; - } + if (contains(flags, flag)) { + return true; } } return false; } + /** + * Checks one flag set for a flag. An undeclared flag, encoded as {@code 0}, is + * carried by no entry. + * + * @param flags One entry's flag set. + * @param flag The flag to look for. + * @return {@code true} if the set contains the flag. + */ + private static boolean contains(int[] flags, int flag) { + if (flag == 0) { + return false; + } + for (final int candidate : flags) { + if (candidate == flag) { + return true; + } + } + return false; + } + + /** + * Checks whether a listed word is valid on its own: some homonym's flag set carries + * none of the blocking flags. An entry whose every flag set is marked + * {@code NEEDAFFIX} is a virtual stem that exists only to be affixed, one marked + * {@code ONLYINCOMPOUND} appears only inside compounds, and one marked + * {@code FORBIDDENWORD} is listed to be blocked; none of them is a word by itself. + * + * @param flagSets The word's flag sets from {@link #lookup(String)}. + * @return {@code true} if some homonym stands on its own. + */ + boolean validStandalone(List flagSets) { + for (final int[] flags : flagSets) { + if (!contains(flags, needAffix) && !contains(flags, onlyInCompound) + && !contains(flags, forbiddenWord)) { + return true; + } + } + return false; + } + + /** + * Checks whether some homonym supports an affix analysis: its flag set carries the + * affix's flag and is neither compound-only nor forbidden. A {@code NEEDAFFIX} set + * does support the analysis, because the removed affix is exactly what the virtual + * stem needs. + * + * @param flagSets The stem's flag sets from {@link #lookup(String)}. + * @param flag The removed affix's flag. + * @return {@code true} if some homonym carries the flag and may stand affixed. + */ + boolean supports(List flagSets, int flag) { + for (final int[] flags : flagSets) { + if (contains(flags, flag) && !contains(flags, onlyInCompound) + && !contains(flags, forbiddenWord)) { + return true; + } + } + return false; + } + + /** + * Checks whether some homonym supports a cross-product analysis: one flag set + * carries both removed affixes' flags and is neither compound-only nor forbidden. + * The two flags must sit in the same set, because homonyms are separate words and + * each removal must be licensed by the same one. + * + * @param flagSets The stem's flag sets from {@link #lookup(String)}. + * @param prefixFlag The removed prefix's flag. + * @param suffixFlag The removed suffix's flag. + * @return {@code true} if some homonym carries both flags and may stand affixed. + */ + boolean supports(List flagSets, int prefixFlag, int suffixFlag) { + for (final int[] flags : flagSets) { + if (contains(flags, prefixFlag) && contains(flags, suffixFlag) + && !contains(flags, onlyInCompound) && !contains(flags, forbiddenWord)) { + return true; + } + } + return false; + } + + /** + * Checks whether a form made with this affix alone is still a virtual stem: the + * affix carries the {@code NEEDAFFIX} flag among its continuation classes, so a + * further affix must join before the form is a word. + * + * @param affix The affix rule to inspect. + * @return {@code true} if the affix alone does not finish a word. + */ + boolean needsFurtherAffix(Affix affix) { + return needAffix != 0 && affix.allowsContinuation(needAffix); + } + + /** + * Checks whether an affix applies only inside compounds: it carries the + * {@code ONLYINCOMPOUND} flag among its continuation classes. + * + * @param affix The affix rule to inspect. + * @return {@code true} if the affix never applies to a standalone word. + */ + boolean compoundOnly(Affix affix) { + return onlyInCompound != 0 && affix.allowsContinuation(onlyInCompound); + } + + /** + * Checks whether an affix is one half of a circumfix: it carries the + * {@code CIRCUMFIX} flag among its continuation classes, so it is only valid on a + * word that also carries a circumfix-marked affix of the other kind, the German + * {@code ge...t} participle being the model. + * + * @param affix The affix rule to inspect. + * @return {@code true} if the affix never applies without its other half. + */ + boolean circumfixOnly(Affix affix) { + return circumfix != 0 && affix.allowsContinuation(circumfix); + } + /** * Reads a stream fully into memory. The stream is not closed. * @@ -354,14 +617,25 @@ private static final class AffixFile { private int compoundBegin; private int compoundEnd; private int compoundMin = 3; + private int needAffix; + private int onlyInCompound; + private int forbiddenWord; + private int circumfix; + private int compoundMiddle; + private int compoundPermit; + private int compoundForbid; + private int compoundWordMax; + private boolean checkCompoundDup; + private boolean checkCompoundCase; + private boolean checkCompoundTriple; } /** * Parses the affix file: the {@code FLAG} declaration, the {@code AF} flag alias - * table, and the {@code PFX} and {@code SFX} blocks. Directives outside the - * supported set (compounding, conversion tables, suggestion options, ...) are - * skipped, so their rules never fire and unsupported analyses are missed rather - * than invented. + * table, the compound and blocking flag declarations, and the {@code PFX} and + * {@code SFX} blocks. Directives outside the supported set (conversion tables, + * suggestion options, the remaining compound machinery, ...) are skipped, so their + * rules never fire and unsupported analyses are missed rather than invented. * * @param content The decoded affix file content. * @return The parsed rules and flag mode. Never {@code null}. @@ -393,29 +667,62 @@ private static AffixFile parseAffix(String content) throws IOException { break; case "COMPOUNDFLAG": case "COMPOUNDBEGIN": + case "COMPOUNDMIDDLE": case "COMPOUNDEND": + case "COMPOUNDPERMITFLAG": + case "COMPOUNDFORBIDFLAG": + case "NEEDAFFIX": + case "PSEUDOROOT": + case "ONLYINCOMPOUND": + case "FORBIDDENWORD": + case "CIRCUMFIX": if (fields.length < 2) { throw new IOException(fields[0] + " line without a flag at line " + (i + 1)); } - final int compound = parseFlag(fields[1], result.flagMode, i + 1); + final int declared = parseFlag(fields[1], result.flagMode, i + 1); switch (fields[0]) { - case "COMPOUNDFLAG" -> result.compoundFlag = compound; - case "COMPOUNDBEGIN" -> result.compoundBegin = compound; - default -> result.compoundEnd = compound; + case "COMPOUNDFLAG" -> result.compoundFlag = declared; + case "COMPOUNDBEGIN" -> result.compoundBegin = declared; + case "COMPOUNDMIDDLE" -> result.compoundMiddle = declared; + case "COMPOUNDEND" -> result.compoundEnd = declared; + case "COMPOUNDPERMITFLAG" -> result.compoundPermit = declared; + case "COMPOUNDFORBIDFLAG" -> result.compoundForbid = declared; + // PSEUDOROOT is the directive's name before hunspell renamed it + case "NEEDAFFIX", "PSEUDOROOT" -> result.needAffix = declared; + case "ONLYINCOMPOUND" -> result.onlyInCompound = declared; + case "CIRCUMFIX" -> result.circumfix = declared; + default -> result.forbiddenWord = declared; } i++; break; case "COMPOUNDMIN": + case "COMPOUNDWORDMAX": if (fields.length < 2) { - throw new IOException("COMPOUNDMIN line without a value at line " + (i + 1)); + throw new IOException(fields[0] + " line without a value at line " + (i + 1)); } try { - result.compoundMin = Math.max(1, Integer.parseInt(fields[1])); + if ("COMPOUNDMIN".equals(fields[0])) { + result.compoundMin = Math.max(1, Integer.parseInt(fields[1])); + } else { + result.compoundWordMax = Math.max(0, Integer.parseInt(fields[1])); + } } catch (NumberFormatException e) { - throw new IOException("malformed COMPOUNDMIN at line " + (i + 1), e); + throw new IOException("malformed " + fields[0] + " at line " + (i + 1), e); } i++; break; + case "CHECKCOMPOUNDDUP": + result.checkCompoundDup = true; + i++; + break; + case "CHECKCOMPOUNDCASE": + result.checkCompoundCase = true; + i++; + break; + case "CHECKCOMPOUNDTRIPLE": + result.checkCompoundTriple = true; + i++; + break; case "AF": // the first AF line declares the alias count; every further AF line is one // alias, a flag run whose 1-based position numeric dictionary flags refer to diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 6ac1a48aa0..8703217ad8 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -24,6 +24,7 @@ import opennlp.tools.stemmer.Stemmer; import opennlp.tools.stemmer.hunspell.HunspellDictionary.Affix; +import opennlp.tools.stemmer.hunspell.HunspellDictionary.CompoundPosition; import opennlp.tools.util.StringUtil; /** @@ -35,7 +36,10 @@ * dictionary entry; {@link #stemAll(CharSequence)} returns every distinct analysis. A * word with no analysis is returned unchanged, so the stemmer degrades to identity on * unknown vocabulary. A form containing uppercase characters is also analyzed in its - * lowercase variant, so sentence-initial capitalization does not hide an entry.

+ * lowercase variant, so sentence-initial capitalization does not hide an entry. + * Entries the dictionary marks as virtual stems ({@code NEEDAFFIX}), compound-only + * parts ({@code ONLYINCOMPOUND}), or forbidden words ({@code FORBIDDENWORD}) never + * count as standalone analyses, matching how hunspell reads those flags.

* *

The {@link Stemmer} interface leaves thread safety to the implementation. This * implementation reads only the immutable dictionary state, so a single instance is @@ -83,7 +87,7 @@ public List stemAll(CharSequence word) { } if (analyses.isEmpty() && dictionary.compoundsDeclared()) { for (final String variant : variants(surface)) { - decompose(variant, analyses); + decompose(variant, surface, analyses); } } if (analyses.isEmpty()) { @@ -116,7 +120,8 @@ private static List variants(String surface) { * @param analyses The mutable, insertion-ordered set collecting the stems found. */ private void analyze(String word, Set analyses) { - if (dictionary.lookup(word) != null) { + final List own = dictionary.lookup(word); + if (own != null && dictionary.validStandalone(own)) { analyses.add(word); } // Only rules whose affix material ends in the word's last character can be @@ -137,50 +142,314 @@ private void analyze(String word, Set analyses) { } /** - * Decomposes a word into two listed compound parts when the affix analysis found - * nothing: at every split point that leaves both sides at least the declared - * minimum length, the left side must be listed and allowed to open a compound and - * the right side listed and allowed to close one. The parts of the first splitting - * that succeeds are reported left to right, so the head-most material comes last, - * and further splittings add any parts not already reported. + * The most part-licensing attempts one decomposition search may spend. Compounding + * searches every split of every tail, which on adversarial input with a + * one-character minimum part length grows without useful bound; the budget stops + * the search there, missing analyses rather than stalling, in line with the + * engine's fail-closed posture. + */ + private static final int PART_CHECK_BUDGET = 2048; + + /** + * Decomposes a word into listed compound parts when the affix analysis found + * nothing: the first part must be admitted to open a compound, every further part + * to continue or close one, each at least the declared minimum length and counted + * against the declared maximum. A part stands on its own entry or on an entry plus + * one affix, the way published dictionaries position their linking forms through + * zero or dash suffixes. The stems of the parts of every successful splitting are + * reported left to right, so the head-most material comes last. A word the + * dictionary lists as forbidden never decomposes; that is how one specific + * ill-formed compound is blocked while its parts stay productive. * * @param word The case variant to decompose. - * @param analyses The mutable, insertion-ordered set collecting the parts. + * @param surface The surface form the variant was derived from; character case at + * junctions is judged against it, so lowercasing a variant cannot + * sidestep a {@code CHECKCOMPOUNDCASE} declaration. + * @param analyses The mutable, insertion-ordered set collecting the part stems. + */ + private void decompose(String word, String surface, Set analyses) { + final List own = dictionary.lookup(word); + if (own != null && dictionary.anyForbidden(own)) { + return; + } + if (word.length() < 2 * dictionary.compoundMin()) { + return; + } + // lowercasing may change the length in exceptional mappings, in which case the + // offsets no longer align and the variant itself is the only usable case source + final String caseSource = surface.length() == word.length() ? surface : word; + search(word, caseSource, 0, new ArrayList<>(), new ArrayList<>(), analyses, + new int[] {PART_CHECK_BUDGET}); + } + + /** + * Extends a partial decomposition with the part starting at {@code from}, trying + * every admissible length and recursing on the remainder. The boundary into this + * part honors the {@code CHECKCOMPOUNDCASE} and {@code CHECKCOMPOUNDTRIPLE} + * declarations, a part repeating its left neighbor honors + * {@code CHECKCOMPOUNDDUP}, and a completed decomposition flushes every part's + * stems into the analyses in part order. + * + * @param word The case variant under decomposition. + * @param caseSource The character-case source for junction checks, the surface + * form when its offsets align with the variant. + * @param from The index the next part starts at. + * @param surfaces The surface strings of the parts taken so far. + * @param stems The licensed stems of the parts taken so far, one list per part. + * @param analyses The mutable, insertion-ordered set collecting the part stems. + * @param budget The remaining part-licensing attempts, counted down in place. */ - private void decompose(String word, Set analyses) { + private void search(String word, String caseSource, int from, List surfaces, + List> stems, Set analyses, int[] budget) { + if (from > 0 && violatesBoundaryChecks(word, caseSource, from)) { + return; + } final int min = dictionary.compoundMin(); - for (int split = min; split <= word.length() - min; split++) { - final String left = word.substring(0, split); - final List leftFlags = dictionary.lookup(left); - if (leftFlags == null || !dictionary.mayBeginCompound(leftFlags)) { - continue; + final int max = dictionary.compoundWordMax(); + final boolean first = from == 0; + // every split leaving room for a further part; a first-position part must also + // leave the closing part, so the whole word is never one part + if (max == 0 || surfaces.size() + 2 <= max) { + for (int end = from + min; end <= word.length() - min; end++) { + if (budget[0] <= 0) { + return; + } + budget[0]--; + final String part = word.substring(from, end); + if (duplicatesNeighbor(part, surfaces)) { + continue; + } + final List partStems = partStems(part, + first ? CompoundPosition.BEGIN : CompoundPosition.MIDDLE, first, false); + if (partStems.isEmpty()) { + continue; + } + surfaces.add(part); + stems.add(partStems); + search(word, caseSource, end, surfaces, stems, analyses, budget); + surfaces.remove(surfaces.size() - 1); + stems.remove(stems.size() - 1); } - final String right = word.substring(split); - final List rightFlags = dictionary.lookup(right); - if (rightFlags == null || !dictionary.mayEndCompound(rightFlags)) { - continue; + } + // the closing part takes the whole remainder; a compound has at least two parts + if (first || word.length() - from < min + || (max > 0 && surfaces.size() + 1 > max) || budget[0] <= 0) { + return; + } + budget[0]--; + final String part = word.substring(from); + if (duplicatesNeighbor(part, surfaces)) { + return; + } + final List partStems = partStems(part, CompoundPosition.END, false, true); + if (partStems.isEmpty()) { + return; + } + for (final List earlier : stems) { + analyses.addAll(earlier); + } + analyses.addAll(partStems); + } + + /** + * Applies the {@code CHECKCOMPOUNDDUP} declaration: a part must not repeat the + * part directly before it. + * + * @param part The candidate part. + * @param surfaces The surface strings of the parts taken so far. + * @return {@code true} if the declaration forbids this part here. + */ + private boolean duplicatesNeighbor(String part, List surfaces) { + return dictionary.checkCompoundDup() && !surfaces.isEmpty() + && part.equals(surfaces.get(surfaces.size() - 1)); + } + + /** + * Applies the character-level boundary declarations at the junction before + * {@code from}: {@code CHECKCOMPOUNDCASE} forbids an uppercase character on either + * side of the junction, and {@code CHECKCOMPOUNDTRIPLE} forbids the same character + * three times in a row across it. + * + * @param word The case variant under decomposition. + * @param caseSource The character-case source for the uppercase judgment. + * @param from The index the junction sits before; greater than zero. + * @return {@code true} if a declaration forbids this junction. + */ + private boolean violatesBoundaryChecks(String word, String caseSource, int from) { + final char before = word.charAt(from - 1); + final char after = word.charAt(from); + if (dictionary.checkCompoundCase() + && (Character.isUpperCase(caseSource.charAt(from - 1)) + || Character.isUpperCase(caseSource.charAt(from)))) { + return true; + } + if (dictionary.checkCompoundTriple() && before == after + && ((from >= 2 && word.charAt(from - 2) == after) + || (from + 1 < word.length() && word.charAt(from + 1) == after))) { + return true; + } + return false; + } + + /** + * Collects the listed stems that admit one part at its compound position: the part + * as its own entry, or an entry plus one suffix or one prefix whose removal leaves + * a listed stem, zero-material rules included, because published dictionaries + * position their linking forms through zero and dash suffixes. An affix at a + * compound-internal boundary must carry the permit flag, a suffix facing the next + * part or a prefix facing the previous one. A part not found as written is also + * tried with its first letter uppercased, the way nouns listed capitalized appear + * lowercase inside a compound. + * + * @param part The part's surface text. + * @param position The part's place in the compound. + * @param first Whether the part opens the word. + * @param last Whether the part closes the word. + * @return The stems admitting the part, in discovery order. Never {@code null}. + */ + private List partStems(String part, CompoundPosition position, + boolean first, boolean last) { + final Set stems = new LinkedHashSet<>(); + collectPartStems(part, position, first, last, stems); + if (stems.isEmpty() && !part.isEmpty()) { + final int initial = part.codePointAt(0); + final int upper = Character.toUpperCase(initial); + if (upper != initial) { + collectPartStems(new StringBuilder().appendCodePoint(upper) + .append(part, Character.charCount(initial), part.length()).toString(), + position, first, last, stems); } - analyses.add(left); - analyses.add(right); + } + return List.copyOf(stems); + } + + /** + * Collects the stems admitting one spelling of a part, bare and through one affix. + * + * @param part The part spelling to look up. + * @param position The part's place in the compound. + * @param first Whether the part opens the word. + * @param last Whether the part closes the word. + * @param stems The mutable, insertion-ordered set collecting the stems. + */ + private void collectPartStems(String part, CompoundPosition position, + boolean first, boolean last, Set stems) { + final List own = dictionary.lookup(part); + if (own != null && dictionary.mayStand(own, position)) { + stems.add(part); + } + for (final Affix suffix : dictionary.suffixesEndingWith(part.charAt(part.length() - 1))) { + collectSuffixedPartStem(part, suffix, position, last, stems); + } + for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { + collectSuffixedPartStem(part, suffix, position, last, stems); + } + for (final Affix prefix : dictionary.prefixesStartingWith(part.charAt(0))) { + collectPrefixedPartStem(part, prefix, position, first, stems); + } + for (final Affix prefix : dictionary.prefixesWithoutMaterial()) { + collectPrefixedPartStem(part, prefix, position, first, stems); + } + } + + /** + * Adds the stem of one suffixed part reading when the rule and the stem's entry + * admit it at the position. + * + * @param part The part spelling under analysis. + * @param suffix The suffix rule to undo. + * @param position The part's place in the compound. + * @param last Whether the part closes the word. + * @param stems The mutable, insertion-ordered set collecting the stems. + */ + private void collectSuffixedPartStem(String part, Affix suffix, + CompoundPosition position, boolean last, Set stems) { + if (dictionary.circumfixOnly(suffix) || dictionary.forbidsInCompound(suffix) + || (!last && !dictionary.permitsInside(suffix))) { + return; + } + final String stem = removeAffixInCompound(part, suffix, true); + if (stem == null) { + return; + } + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && dictionary.supportsPart(flagSets, suffix.flag(), position, + dictionary.affixAdmits(suffix, position))) { + stems.add(stem); + } + } + + /** + * Adds the stem of one prefixed part reading when the rule and the stem's entry + * admit it at the position. + * + * @param part The part spelling under analysis. + * @param prefix The prefix rule to undo. + * @param position The part's place in the compound. + * @param first Whether the part opens the word. + * @param stems The mutable, insertion-ordered set collecting the stems. + */ + private void collectPrefixedPartStem(String part, Affix prefix, + CompoundPosition position, boolean first, Set stems) { + if (dictionary.circumfixOnly(prefix) || dictionary.forbidsInCompound(prefix) + || (!first && !dictionary.permitsInside(prefix))) { + return; + } + final String stem = removeAffixInCompound(part, prefix, false); + if (stem == null) { + return; + } + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && dictionary.supportsPart(flagSets, prefix.flag(), position, + dictionary.affixAdmits(prefix, position))) { + stems.add(stem); } } + /** + * Undoes one affix rule on a compound part. Unlike the standalone removals, a rule + * that neither adds nor removes material is undone here, to its own spelling with + * the condition checked, because dictionaries position compound parts through + * exactly such zero rules. + * + * @param part The part spelling under analysis. + * @param affix The rule to undo. + * @param suffix Whether the rule is a suffix rule. + * @return The candidate stem, or {@code null} when the rule does not apply. + */ + private static String removeAffixInCompound(String part, Affix affix, boolean suffix) { + if (affix.affix().isEmpty() && affix.strip().isEmpty()) { + return affix.condition().matches(part) ? part : null; + } + return suffix ? removeSuffix(part, affix) : removePrefix(part, affix); + } + /** * Undoes one suffix rule and, through continuation classes, one further suffix on - * the intermediate stem, adding every dictionary-confirmed analysis. + * the intermediate stem, adding every dictionary-confirmed analysis. A rule that + * applies only inside compounds or only as half of a circumfix is not undone at all, + * the latter because no prefix accompanies it on this path; a rule marked as needing + * a further affix yields no single-removal analysis, because the surface form it + * makes alone is a virtual stem; its twofold analyses stand, the inner affix being + * exactly the further one required. * * @param word The case variant under analysis. * @param suffix The suffix rule to undo. * @param analyses The mutable, insertion-ordered set collecting the stems found. */ private void undoSuffix(String word, Affix suffix, Set analyses) { + if (dictionary.compoundOnly(suffix) || dictionary.circumfixOnly(suffix)) { + return; + } final String stem = removeSuffix(word, suffix); if (stem == null) { return; } - final List flagSets = dictionary.lookup(stem); - if (flagSets != null && HunspellDictionary.hasFlag(flagSets, suffix.flag())) { - analyses.add(stem); + if (!dictionary.needsFurtherAffix(suffix)) { + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && dictionary.supports(flagSets, suffix.flag())) { + analyses.add(stem); + } } for (final Affix inner : dictionary.suffixesEndingWith(stem.charAt(stem.length() - 1))) { undoInnerSuffix(stem, suffix, inner, analyses); @@ -201,7 +470,8 @@ private void undoSuffix(String word, Affix suffix, Set analyses) { */ private void undoInnerSuffix(String stem, Affix outer, Affix inner, Set analyses) { - if (!inner.allowsContinuation(outer.flag())) { + if (!inner.allowsContinuation(outer.flag()) || dictionary.compoundOnly(inner) + || dictionary.circumfixOnly(inner)) { return; } final String doubleStem = removeSuffix(stem, inner); @@ -209,27 +479,36 @@ private void undoInnerSuffix(String stem, Affix outer, Affix inner, return; } final List innerFlags = dictionary.lookup(doubleStem); - if (innerFlags != null && HunspellDictionary.hasFlag(innerFlags, inner.flag())) { + if (innerFlags != null && dictionary.supports(innerFlags, inner.flag())) { analyses.add(doubleStem); } } /** * Undoes one prefix rule and, for cross-product rules, one further suffix on the - * intermediate stem, adding every dictionary-confirmed analysis. + * intermediate stem, adding every dictionary-confirmed analysis. A rule that + * applies only inside compounds is not undone at all. A rule marked as needing a + * further affix or as half of a circumfix yields no single-removal analysis; its + * cross-product analyses stand, the suffix being exactly the further affix or the + * other circumfix half required. * * @param word The case variant under analysis. * @param prefix The prefix rule to undo. * @param analyses The mutable, insertion-ordered set collecting the stems found. */ private void undoPrefix(String word, Affix prefix, Set analyses) { + if (dictionary.compoundOnly(prefix)) { + return; + } final String stem = removePrefix(word, prefix); if (stem == null) { return; } - final List flagSets = dictionary.lookup(stem); - if (flagSets != null && HunspellDictionary.hasFlag(flagSets, prefix.flag())) { - analyses.add(stem); + if (!dictionary.needsFurtherAffix(prefix) && !dictionary.circumfixOnly(prefix)) { + final List flagSets = dictionary.lookup(stem); + if (flagSets != null && dictionary.supports(flagSets, prefix.flag())) { + analyses.add(stem); + } } if (!prefix.crossProduct()) { return; @@ -243,7 +522,10 @@ private void undoPrefix(String word, Affix prefix, Set analyses) { } /** - * Undoes the suffix half of a cross-product removal when both rules opted in. + * Undoes the suffix half of a cross-product removal when both rules opted in. The + * two rules must agree on circumfixing: a circumfix-marked affix is only valid with + * a marked affix of the other kind, so a pair of which exactly one is marked mixes + * an ordinary affix into a circumfix and is rejected. * * @param stem The intermediate stem after the prefix removal. * @param prefix The already-undone prefix rule. @@ -252,16 +534,18 @@ private void undoPrefix(String word, Affix prefix, Set analyses) { */ private void undoCrossProductSuffix(String stem, Affix prefix, Affix suffix, Set analyses) { - if (!suffix.crossProduct()) { + if (!suffix.crossProduct() || dictionary.compoundOnly(suffix) + || dictionary.circumfixOnly(prefix) != dictionary.circumfixOnly(suffix)) { return; } final String doubleStem = removeSuffix(stem, suffix); if (doubleStem == null) { return; } + // a needs-further-affix marker on either rule is satisfied by the other rule, + // so no such check applies here; both flags must sit in one homonym's flag set final List both = dictionary.lookup(doubleStem); - if (both != null && HunspellDictionary.hasFlag(both, prefix.flag()) - && HunspellDictionary.hasFlag(both, suffix.flag())) { + if (both != null && dictionary.supports(both, prefix.flag(), suffix.flag())) { analyses.add(doubleStem); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java new file mode 100644 index 0000000000..d6d12faa44 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +/** + * Gated checks against published dictionaries, which are never bundled: the tests run + * only when {@code -Dopennlp.hunspell.dict.dir} names a directory holding + * {@code .aff}/{@code .dic} pairs, and each test additionally skips when + * its dictionary pair is absent. The download helper in {@code dev/} fetches the pairs + * together with their license files; see {@code dev/README-hunspell-dictionaries.md}. + * + *

The assertions are limited to morphology stable across dictionary revisions: + * everyday inflections, and for German the decomposability of ordinary compounds.

+ */ +public class HunspellRealDictionaryTest { + + private static final String DICT_DIR_PROPERTY = "opennlp.hunspell.dict.dir"; + + /** + * Loads one dictionary pair from the gated directory, skipping the test when the + * gate or the pair is absent. + * + * @param name The dictionary base name, such as {@code en_US}. + * @return A stemmer over the loaded pair. Never {@code null}. + * @throws IOException Thrown if a present pair fails to load, which is a failure, + * not a skip. + */ + private static HunspellStemmer loadOrSkip(String name) throws IOException { + final String dir = System.getProperty(DICT_DIR_PROPERTY); + Assumptions.assumeTrue(dir != null && !dir.isBlank(), + "no " + DICT_DIR_PROPERTY + " given"); + final Path affix = Path.of(dir, name + ".aff"); + final Path words = Path.of(dir, name + ".dic"); + Assumptions.assumeTrue(Files.isReadable(affix) && Files.isReadable(words), + name + " pair not present under " + dir); + return new HunspellStemmer(HunspellDictionary.load(affix, words)); + } + + @Test + void testEnglishInflections() throws IOException { + final HunspellStemmer stemmer = loadOrSkip("en_US"); + Assertions.assertEquals("worker", stemmer.stem("workers").toString()); + Assertions.assertEquals("cat", stemmer.stem("cats").toString()); + Assertions.assertEquals("unhappy", stemmer.stem("unhappiest").toString()); + Assertions.assertEquals("quick", stemmer.stem("quickly").toString()); + Assertions.assertEquals("look", stemmer.stem("looked").toString()); + // unknown vocabulary degrades to identity + Assertions.assertEquals("zyzzyvax", stemmer.stem("zyzzyvax").toString()); + } + + @Test + void testGermanInflections() throws IOException { + final HunspellStemmer stemmer = loadOrSkip("de_DE_frami"); + Assertions.assertEquals("Kind", stemmer.stem("Kinder").toString()); + Assertions.assertEquals("Haus", stemmer.stem("Häuser").toString()); + Assertions.assertEquals("schnell", stemmer.stem("schnellsten").toString()); + } + + @Test + void testGermanCompoundsDecompose() throws IOException { + final HunspellStemmer stemmer = loadOrSkip("de_DE_frami"); + // the exact part spellings follow the dictionary's own entries and may shift + // between revisions; that ordinary compounds decompose at all must not + Assertions.assertTrue(stemmer.stemAll("Haustür").size() >= 2); + Assertions.assertTrue(stemmer.stemAll("Kinderzimmer").size() >= 2); + Assertions.assertTrue(stemmer.stemAll("Abbildungsverzeichnis").size() >= 2); + } + + @Test + void testHungarianInflections() throws IOException { + final HunspellStemmer stemmer = loadOrSkip("hu_HU"); + Assertions.assertEquals("kutya", stemmer.stem("kutyák").toString()); + Assertions.assertEquals("asztal", stemmer.stem("asztalon").toString()); + Assertions.assertEquals("könyv", stemmer.stem("könyveket").toString()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 9ffd1289b2..d4fe1a1961 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -775,4 +775,383 @@ void testCompoundMinBoundsThePartLength() throws IOException { // the left side would be three characters, below the declared minimum of four Assertions.assertEquals(List.of("doghouse"), stemmer.stemAll("doghouse")); } + + /** + * Verifies the NEEDAFFIX flag on entries: a virtual stem exists only to be affixed, + * the linking forms of the published German dictionary being the model, so its bare + * form is no analysis of itself while its affixed forms still reduce to it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testNeedAffixEntryIsNoStandaloneAnalysis() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "NEEDAFFIX h\nSFX S Y 1\nSFX S 0 s .\nSFX K Y 1\nSFX K 0 k .\n", + "2\nlink/hS\nlin/K\n")); + // the virtual entry no longer explains the bare form; only the k analysis remains + Assertions.assertEquals(List.of("lin"), stemmer.stemAll("link")); + // affixed, the virtual stem is exactly what the s removal lands on + Assertions.assertEquals(List.of("link"), stemmer.stemAll("links")); + } + + /** + * Verifies NEEDAFFIX against homonyms: the flag blocks one entry's flag set, not + * the word, so a second listing without the flag keeps the bare form valid. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testNeedAffixHomonymKeepsTheBareWord() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "NEEDAFFIX h\nSFX S Y 1\nSFX S 0 s .\n", + "2\nlink/hS\nlink\n")); + Assertions.assertEquals(List.of("link"), stemmer.stemAll("link")); + } + + /** + * Verifies the historical PSEUDOROOT alias, the directive's name before hunspell + * renamed it to NEEDAFFIX; older dictionaries still declare it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testPseudoRootIsNeedAffixByItsOldName() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "PSEUDOROOT h\nSFX S Y 1\nSFX S 0 s .\nSFX K Y 1\nSFX K 0 k .\n", + "2\nlink/hS\nlin/K\n")); + Assertions.assertEquals(List.of("lin"), stemmer.stemAll("link")); + } + + /** + * Verifies the NEEDAFFIX flag on affix rules: a rule carrying the flag among its + * continuation classes makes a form that still needs another affix, so its + * single-removal analysis is suppressed while a twofold removal, whose inner affix + * is the further one required, still reports the stem. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testNeedAffixOnAnAffixRequiresAnotherAffix() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + String.join("\n", + "NEEDAFFIX h", + "SFX A Y 1", + "SFX A 0 er/hB .", + "SFX B Y 1", + "SFX B 0 s .", + ""), + "1\nwork/A\n")); + // work + er alone is virtual, so worker has no analysis and passes through + Assertions.assertEquals(List.of("worker"), stemmer.stemAll("worker")); + // work + er + s is complete; the twofold removal reaches the listed stem + Assertions.assertEquals(List.of("work"), stemmer.stemAll("workers")); + } + + /** + * Verifies that a cross-product analysis satisfies an affix's NEEDAFFIX marker: + * the prefix is the further affix the marked suffix requires, mirroring how + * hunspell accepts a prefix plus a needs-affix suffix together. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCrossProductSatisfiesNeedAffixOnTheSuffix() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + String.join("\n", + "NEEDAFFIX h", + "PFX P Y 1", + "PFX P 0 un .", + "SFX A Y 1", + "SFX A 0 er/h .", + ""), + "1\nwork/AP\n")); + Assertions.assertEquals(List.of("worker"), stemmer.stemAll("worker")); + Assertions.assertEquals(List.of("work"), stemmer.stemAll("unworker")); + } + + /** + * Verifies the ONLYINCOMPOUND flag: an entry carrying it appears only inside + * compounds, the ordinal parts of the published US English dictionary being the + * model, so neither its bare form nor its affixed forms are standalone analyses, + * while compound decomposition may still use it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testOnlyInCompoundEntrySupportsNoStandaloneAnalyses() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "ONLYINCOMPOUND c\nCOMPOUNDFLAG Z\nCOMPOUNDMIN 3\nSFX S Y 1\nSFX S 0 s .\n", + "3\npart/cSZ\nhouse/Z\nwalk/S\n")); + // the affix analysis is suppressed because part's only flag set is compound-only + Assertions.assertEquals(List.of("parts"), stemmer.stemAll("parts")); + Assertions.assertEquals(List.of("part"), stemmer.stemAll("part")); + // inside a compound the entry serves exactly its declared purpose + Assertions.assertEquals(List.of("part", "house"), stemmer.stemAll("parthouse")); + Assertions.assertEquals(List.of("walk"), stemmer.stemAll("walks")); + } + + /** + * Verifies the FORBIDDENWORD flag: an entry carrying it is listed to be blocked, + * so it supports no analysis and no compound part. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testForbiddenWordSupportsNothing() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "FORBIDDENWORD w\nCOMPOUNDFLAG Z\nCOMPOUNDMIN 3\nSFX S Y 1\nSFX S 0 s .\n", + "3\nfoo/wSZ\nhouse/Z\nbar/S\n")); + Assertions.assertEquals(List.of("foo"), stemmer.stemAll("foo")); + Assertions.assertEquals(List.of("foos"), stemmer.stemAll("foos")); + Assertions.assertEquals(List.of("foohouse"), stemmer.stemAll("foohouse")); + Assertions.assertEquals(List.of("bar"), stemmer.stemAll("bars")); + } + + /** The circumfix fixture: the German {@code ge...t} participle in miniature. */ + private static final String CIRCUMFIX_AFFIX = String.join("\n", + "CIRCUMFIX f", + "PFX G Y 1", + "PFX G 0 ge/f .", + "SFX T Y 1", + "SFX T en et/f en", + "PFX U Y 1", + "PFX U 0 un .", + "SFX S Y 1", + "SFX S 0 s .", + ""); + + /** + * Verifies the CIRCUMFIX flag: two marked halves analyze together and neither + * analyzes alone, so the participle reduces to its verb while the half-applied + * forms stay unexplained. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCircumfixHalvesOnlyAnalyzeTogether() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + CIRCUMFIX_AFFIX, "1\narbeiten/GT\n")); + Assertions.assertEquals(List.of("arbeiten"), stemmer.stemAll("gearbeitet")); + // the suffix half alone is no word, although the stem carries its flag + Assertions.assertEquals(List.of("arbeitet"), stemmer.stemAll("arbeitet")); + // the prefix half alone is no word either + Assertions.assertEquals(List.of("gearbeiten"), stemmer.stemAll("gearbeiten")); + } + + /** + * Verifies that circumfixing rejects mixed pairs: a marked half never combines + * with an unmarked affix of the other kind, in either direction, while a fully + * unmarked cross-product in the same dictionary still analyzes. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCircumfixRejectsMixedPairs() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + CIRCUMFIX_AFFIX, "2\narbeiten/GTUS\nlauf/US\n")); + // unmarked prefix with the marked suffix half + Assertions.assertEquals(List.of("unarbeitet"), stemmer.stemAll("unarbeitet")); + // the marked prefix half with an unmarked suffix + Assertions.assertEquals(List.of("gearbeitens"), stemmer.stemAll("gearbeitens")); + // both halves marked still analyze beside the rejected mixtures + Assertions.assertEquals(List.of("arbeiten"), stemmer.stemAll("gearbeitet")); + // a fully unmarked cross-product is untouched by the circumfix declaration + Assertions.assertEquals(List.of("lauf"), stemmer.stemAll("unlaufs")); + } + + /** + * Verifies decomposition beyond two parts: the positional flags admit a begin, a + * middle, and an end part, a part fit only for the middle neither opens nor closes, + * and repeated middles fold into the reported set. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundMiddleAdmitsInnerParts() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDBEGIN B\nCOMPOUNDMIDDLE M\nCOMPOUNDEND E\nCOMPOUNDMIN 3\n", + "3\ndog/B\ncat/M\nhouse/E\n")); + Assertions.assertEquals(List.of("dog", "cat", "house"), + stemmer.stemAll("dogcathouse")); + Assertions.assertEquals(List.of("dog", "cat", "house"), + stemmer.stemAll("dogcatcathouse")); + Assertions.assertEquals(List.of("dog", "house"), stemmer.stemAll("doghouse")); + // cat holds only the middle flag, so it neither opens nor closes + Assertions.assertEquals(List.of("cathouse"), stemmer.stemAll("cathouse")); + Assertions.assertEquals(List.of("dogcat"), stemmer.stemAll("dogcat")); + } + + /** + * Verifies COMPOUNDWORDMAX: a decomposition needing more parts than declared is + * rejected while one within the bound still analyzes. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundWordMaxBoundsThePartCount() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDBEGIN B\nCOMPOUNDMIDDLE M\nCOMPOUNDEND E\nCOMPOUNDMIN 3\n" + + "COMPOUNDWORDMAX 2\n", + "3\ndog/B\ncat/M\nhouse/E\n")); + Assertions.assertEquals(List.of("dog", "house"), stemmer.stemAll("doghouse")); + Assertions.assertEquals(List.of("dogcathouse"), stemmer.stemAll("dogcathouse")); + } + + /** + * Verifies affixed compound parts, the German linking form being the model: a part + * is its entry plus one suffix whose continuation classes position the derived form + * and permit it at the internal boundary, and the reported analysis is the entry, + * not the linking form. The lowercase interior spelling of a capitalized entry is + * found through the part's uppercased retry. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testLinkingSuffixJoinsCompoundParts() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + String.join("\n", + "COMPOUNDBEGIN x", + "COMPOUNDEND z", + "COMPOUNDPERMITFLAG c", + "COMPOUNDMIN 2", + "SFX j Y 1", + "SFX j 0 s/xc .", + ""), + "2\nAbbildung/j\nVerzeichnis/z\n")); + Assertions.assertEquals(List.of("Abbildung", "Verzeichnis"), + stemmer.stemAll("Abbildungsverzeichnis")); + // without the linking s the first part has no admitting reading + Assertions.assertEquals(List.of("Abbildungverzeichnis"), + stemmer.stemAll("Abbildungverzeichnis")); + } + + /** + * Verifies zero-suffix part positioning, the pattern the published German + * dictionary uses: a virtual stem enters compounds through a rule that adds no + * material but whose continuation classes carry the positional and permit flags. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testZeroSuffixPositionsAVirtualStemInCompounds() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + String.join("\n", + "NEEDAFFIX h", + "COMPOUNDBEGIN x", + "COMPOUNDEND z", + "COMPOUNDPERMITFLAG c", + "COMPOUNDMIN 3", + "SFX j Y 1", + "SFX j 0 0/xc .", + ""), + "2\nfugen/hj\nwerk/z\n")); + Assertions.assertEquals(List.of("fugen", "werk"), stemmer.stemAll("fugenwerk")); + // the virtual stem alone is still no word + Assertions.assertEquals(List.of("fugen"), stemmer.stemAll("fugen")); + } + + /** + * Verifies COMPOUNDFORBIDFLAG: an affixed form whose rule carries the flag stays + * out of compounds although its positioning otherwise admits it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCompoundForbidFlagBarsAnAffixedPart() throws IOException { + final String words = "2\ndog/ZS\nhouse/Z\n"; + final HunspellStemmer barred = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDPERMITFLAG c\nCOMPOUNDFORBIDFLAG F\nCOMPOUNDMIN 3\n" + + "SFX S Y 1\nSFX S 0 s/cF .\n", + words)); + Assertions.assertEquals(List.of("dogshouse"), barred.stemAll("dogshouse")); + final HunspellStemmer allowed = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDPERMITFLAG c\nCOMPOUNDMIN 3\n" + + "SFX S Y 1\nSFX S 0 s/c .\n", + words)); + Assertions.assertEquals(List.of("dog", "house"), allowed.stemAll("dogshouse")); + } + + /** + * Verifies that an affix without the permit flag keeps off internal boundaries: a + * suffixed reading fits the last part but not an earlier one. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testAffixWithoutPermitFlagStaysAtTheEdge() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\nSFX S Y 1\nSFX S 0 s .\n", + "2\ndog/ZS\nhouse/ZS\n")); + // the suffix closes the word, so the last part may carry it + Assertions.assertEquals(List.of("dog", "house"), stemmer.stemAll("doghouses")); + // an internal suffix without the permit flag blocks the split + Assertions.assertEquals(List.of("dogshouse"), stemmer.stemAll("dogshouse")); + } + + /** + * Verifies CHECKCOMPOUNDDUP: a part must not repeat its left neighbor, while the + * same dictionary without the declaration accepts the repetition. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCheckCompoundDupForbidsRepeatedParts() throws IOException { + final String words = "2\ndog/Z\nhouse/Z\n"; + final HunspellStemmer checked = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\nCHECKCOMPOUNDDUP\n", words)); + Assertions.assertEquals(List.of("dogdoghouse"), checked.stemAll("dogdoghouse")); + final HunspellStemmer unchecked = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\n", words)); + Assertions.assertEquals(List.of("dog", "house"), unchecked.stemAll("dogdoghouse")); + } + + /** + * Verifies CHECKCOMPOUNDCASE: an uppercase character on either side of a junction + * forbids the split, while the same dictionary without the declaration accepts it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCheckCompoundCaseForbidsUppercaseJunctions() throws IOException { + final String words = "2\ndog/Z\nHouse/Z\n"; + final HunspellStemmer checked = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\nCHECKCOMPOUNDCASE\n", words)); + Assertions.assertEquals(List.of("dogHouse"), checked.stemAll("dogHouse")); + final HunspellStemmer unchecked = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\n", words)); + Assertions.assertEquals(List.of("dog", "House"), unchecked.stemAll("dogHouse")); + } + + /** + * Verifies CHECKCOMPOUNDTRIPLE: the same character three times in a row across a + * junction forbids the split, while the same dictionary without the declaration + * accepts it. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testCheckCompoundTripleForbidsTripleLetters() throws IOException { + final String words = "2\nbell/Z\nlow/Z\n"; + final HunspellStemmer checked = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\nCHECKCOMPOUNDTRIPLE\n", words)); + Assertions.assertEquals(List.of("belllow"), checked.stemAll("belllow")); + final HunspellStemmer unchecked = new HunspellStemmer(load( + "COMPOUNDFLAG Z\nCOMPOUNDMIN 3\n", words)); + Assertions.assertEquals(List.of("bell", "low"), unchecked.stemAll("belllow")); + } + + /** + * Verifies that a listed forbidden word never decomposes: the dictionary blocks + * one specific ill-formed compound while its parts stay productive elsewhere. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testForbiddenEntryBlocksItsDecomposition() throws IOException { + final HunspellStemmer stemmer = new HunspellStemmer(load( + "FORBIDDENWORD w\nCOMPOUNDFLAG Z\nCOMPOUNDMIN 3\n", + "4\ndog/Z\nhouse/Z\ncat/Z\ndoghouse/w\n")); + Assertions.assertEquals(List.of("doghouse"), stemmer.stemAll("doghouse")); + Assertions.assertEquals(List.of("cat", "house"), stemmer.stemAll("cathouse")); + } } From 8d5afa2297b060985b6dc20dfe5de10301f5b48b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:38:12 -0400 Subject: [PATCH 11/24] OPENNLP-1893: Add hunspell manual coverage with mirror-tested examples Extend docbkx/stemmer.xml with the hunspell affix-stemmer section, wire the chapter into the manual, and add StemmerFactoryUsageExampleTest and HunspellManualExampleTest asserting the load-and-stem values the chapter prints. Point the dictionary README at the new manual example. --- .../dev/README-hunspell-dictionaries.md | 2 +- .../hunspell/HunspellManualExampleTest.java | 73 +++++++++++++++++++ opennlp-docs/src/docbkx/stemmer.xml | 26 +++++++ 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellManualExampleTest.java diff --git a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md index 457fd01eeb..440814163e 100644 --- a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md +++ b/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md @@ -46,7 +46,7 @@ Stemmer stemmer = factory.newStemmer(); CharSequence stem = stemmer.stem("workers"); ``` -What `stem` evaluates to is decided by the dictionary you loaded, and this project ships no dictionary data, so no result is claimed here for `en_US`. What is verified is the flow above: `HunspellStemmerFactoryTest#testEndToEndUsageFromFiles` runs exactly these calls against a project-authored `.aff`/`.dic` pair written to disk, in which `work` carries the agentive `-er` rule and its continuation class for the plural `-s`, and asserts that `stemmer.stem("workers")` returns `work`. +What `stem` evaluates to is decided by the dictionary you loaded, and this project ships no dictionary data, so no result is claimed here for `en_US`. The same load-and-stem flow is pinned by `HunspellManualExampleTest` (miniature in-memory dictionary, asserted stems for `workers` and `worker`) and by `HunspellStemmerFactoryTest#testEndToEndUsageFromFiles` (the same pair written to disk). The developer manual chapter `stemmer.xml` cites `HunspellManualExampleTest`. The dictionary is immutable and safe to share between threads; the factory hands out a fresh stemmer per call, so each thread takes its own from `newStemmer()`. A dictionary that declares a non-UTF-8 encoding through the `SET` directive in its `.aff` file is decoded accordingly; nothing needs converting beforehand. diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellManualExampleTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellManualExampleTest.java new file mode 100644 index 0000000000..21daec7e0b --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellManualExampleTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.stemmer.Stemmer; + +/** + * Runs the manual's Hunspell examples (docbkx {@code stemmer.xml}) verbatim: every + * value the chapter states is asserted here, so a change breaking this test breaks the + * manual. The fixture dictionary is authored inside this class; no external dictionary + * data is involved. + */ +public class HunspellManualExampleTest { + + /** + * Affix fixture matching the chapter: agentive {@code -er} with continuation class + * {@code S}, and the plural {@code -s}. + */ + private static final String AFFIX = String.join("\n", + "SET UTF-8", + "SFX E Y 1", + "SFX E 0 er/S .", + "SFX S Y 1", + "SFX S 0 s [^sxy]", + ""); + + /** Word-list fixture: {@code work} accepts both suffixes. */ + private static final String WORDS = "1\nwork/ES\n"; + + /** + * Loads the chapter's miniature dictionary, stems through a factory-minted stemmer, + * and asserts the exact stems the manual prints. + * + * @throws IOException Thrown if the in-memory fixture fails to load. + */ + @Test + void testLoadAndStemWorkers() throws IOException { + final HunspellDictionary dictionary = HunspellDictionary.load( + new ByteArrayInputStream(AFFIX.getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(WORDS.getBytes(StandardCharsets.UTF_8))); + final Stemmer stemmer = new HunspellStemmerFactory(dictionary).newStemmer(); + + Assertions.assertEquals("work", stemmer.stem("workers").toString()); + Assertions.assertEquals("work", stemmer.stem("worker").toString()); + Assertions.assertEquals(List.of("work"), + stemmer.stemAll("workers").stream().map(CharSequence::toString).toList()); + // unknown vocabulary passes through unchanged + Assertions.assertEquals("table", stemmer.stem("table").toString()); + } +} diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 248b310c15..8dc078d975 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -69,4 +69,30 @@ new CachingStemmer(factory).stem("running"); // "run"]]> longer uses a sharing or caching stemmer. + +
+ Hunspell dictionaries + + opennlp.tools.stemmer.hunspell is a clean-room engine over the + documented Hunspell dictionary format: a user-supplied + .aff affix file and its .dic word list. OpenNLP + bundles no dictionary data, so each dictionary's own license stays with + the files you download. The dictionary is immutable and safe to share; + HunspellStemmerFactory hands out a fresh stemmer per call. + HunspellManualExampleTest asserts the behavior shown here. + + + The in-tree test uses a project-authored miniature dictionary instead of a + published one, and asserts the same stems for workers and + worker. Acquisition helpers and the supported affix feature + set live in + opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md. + +
From dd0d2d928ee6ba00228f35ebdab0684553adb738 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 06:39:17 -0400 Subject: [PATCH 12/24] OPENNLP-1893: Apply the review-convention pass: factual license prose, split null contracts, thread-safety annotations --- .../README-hunspell-dictionaries.md | 4 +- .../download-hunspell-dictionary.sh | 7 +- .../stemmer/hunspell/HunspellDictionary.java | 97 ++++++++----------- .../stemmer/hunspell/HunspellStemmer.java | 31 +++--- .../hunspell/HunspellStemmerFactory.java | 3 +- .../hunspell/HunspellRealDictionaryTest.java | 12 ++- .../hunspell/HunspellStemmerFactoryTest.java | 19 ++-- .../stemmer/hunspell/HunspellStemmerTest.java | 22 +++-- opennlp-docs/src/docbkx/stemmer.xml | 10 +- 9 files changed, 103 insertions(+), 102 deletions(-) rename {opennlp-core/opennlp-runtime/dev => dev}/README-hunspell-dictionaries.md (85%) rename {opennlp-core/opennlp-runtime/dev => dev}/download-hunspell-dictionary.sh (92%) diff --git a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md b/dev/README-hunspell-dictionaries.md similarity index 85% rename from opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md rename to dev/README-hunspell-dictionaries.md index 440814163e..34cb5c2783 100644 --- a/opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md +++ b/dev/README-hunspell-dictionaries.md @@ -17,11 +17,11 @@ # Hunspell dictionaries for the affix stemmer -The Hunspell stemmer (`opennlp.tools.stemmer.hunspell`) is a clean-room engine over the documented Hunspell dictionary format: a `.dic` word list plus its `.aff` affix companion, both supplied by the user. Apache OpenNLP bundles no dictionary data, so the dictionaries' own licenses never attach to the library; whichever dictionary you download, its license is stated in the readme shipped alongside it and is yours to comply with. +The Hunspell stemmer (`opennlp.tools.stemmer.hunspell`) implements the documented Hunspell dictionary format: a `.dic` word list plus its `.aff` affix companion, both supplied by the user. Apache OpenNLP bundles no dictionary data; whichever dictionary you download, its license is stated in the readme shipped alongside it. ## Where dictionaries come from -The LibreOffice project maintains a large collection of Hunspell dictionaries, one directory per language, at `github.com/LibreOffice/dictionaries`. Licenses differ per dictionary, which is exactly why nothing is bundled: for example, the `en_US` dictionary derives from SCOWL and states its terms in `README_en_US.txt` in the same directory. Many other sources work too; the engine only cares that the pair follows the Hunspell format. +The LibreOffice project maintains a large collection of Hunspell dictionaries, one directory per language, at `github.com/LibreOffice/dictionaries`. Licenses differ per dictionary, which is why nothing is bundled: for example, the `en_US` dictionary derives from SCOWL and states its terms in `README_en_US.txt` in the same directory. Many other sources work too; the engine only cares that the pair follows the Hunspell format. The helper next to this file fetches a pair together with its readme files: diff --git a/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh b/dev/download-hunspell-dictionary.sh similarity index 92% rename from opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh rename to dev/download-hunspell-dictionary.sh index 5ed95511b0..afaff59e60 100755 --- a/opennlp-core/opennlp-runtime/dev/download-hunspell-dictionary.sh +++ b/dev/download-hunspell-dictionary.sh @@ -16,10 +16,9 @@ # Fetches one Hunspell dictionary pair (.aff and .dic) plus its license/readme files # from the LibreOffice dictionaries collection. Each dictionary carries its own -# license, stated in the readme files this script downloads alongside it, and you -# accept that license by using the dictionary. Apache OpenNLP bundles no dictionary -# data. See README-hunspell-dictionaries.md in this directory for the Java steps that -# follow. +# license, stated in the readme files this script downloads alongside it. Apache +# OpenNLP bundles no dictionary data. See README-hunspell-dictionaries.md in this +# directory for the Java steps that follow. set -euo pipefail diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 7197185ffb..5970304a4e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -17,11 +17,12 @@ package opennlp.tools.stemmer.hunspell; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; +import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.StandardCharsets; +import java.nio.charset.UnsupportedCharsetException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -30,14 +31,14 @@ import java.util.List; import java.util.Map; +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.util.StringUtil; /** * An immutable, in-memory Hunspell-format dictionary: the word list of a {@code .dic} * file and the prefix and suffix rules of its {@code .aff} companion, loaded from - * user-supplied files. The engine is a clean-room implementation of the documented - * format; no dictionary data is bundled, so the dictionaries' own licenses never attach - * to this library. + * user-supplied files. The engine implements the documented format directly; no + * dictionary data is bundled, dictionaries are supplied by the user. * *

Supported affix features: {@code PFX} and {@code SFX} rules with strip strings, * character-class conditions, and cross-product combination of one prefix with one @@ -63,8 +64,8 @@ * * @see HunspellStemmer * @see HunspellStemmerFactory - * @since 3.0.0 */ +@ThreadSafe public final class HunspellDictionary { /** @@ -93,17 +94,6 @@ boolean allowsContinuation(int otherFlag) { } } - private final Map> entries; - private final List prefixes; - private final List suffixes; - private final Map> suffixesByLast; - private final List suffixesWithoutMaterial; - private final Map> prefixesByFirst; - private final List prefixesWithoutMaterial; - private final int compoundFlag; - private final int compoundBegin; - private final int compoundEnd; - private final int compoundMin; /** The place a part takes in a compound, deciding which positional flag admits it. */ enum CompoundPosition { /** The first part. */ @@ -114,6 +104,18 @@ enum CompoundPosition { END } + /** The shared empty bucket answered for characters no affix rule is keyed under. */ + private static final List NO_AFFIXES = List.of(); + + private final Map> entries; + private final Map> suffixesByLast; + private final List suffixesWithoutMaterial; + private final Map> prefixesByFirst; + private final List prefixesWithoutMaterial; + private final int compoundFlag; + private final int compoundBegin; + private final int compoundEnd; + private final int compoundMin; private final int needAffix; private final int onlyInCompound; private final int forbiddenWord; @@ -143,8 +145,6 @@ private HunspellDictionary(Map> entries, AffixFile affix) { this.checkCompoundCase = affix.checkCompoundCase; this.checkCompoundTriple = affix.checkCompoundTriple; this.entries = entries; - this.prefixes = List.copyOf(affix.prefixes); - this.suffixes = List.copyOf(affix.suffixes); // Undoing a suffix requires the word to end with the rule's affix material, so // only rules whose material ends in the word's last character can ever apply; // the same holds for prefixes and the first character. Bucketing by that @@ -153,7 +153,7 @@ private HunspellDictionary(Map> entries, AffixFile affix) { // everywhere. this.suffixesByLast = new HashMap<>(); this.suffixesWithoutMaterial = new ArrayList<>(); - for (final Affix suffix : suffixes) { + for (final Affix suffix : affix.suffixes) { final String material = suffix.affix(); if (material.isEmpty()) { suffixesWithoutMaterial.add(suffix); @@ -164,7 +164,7 @@ private HunspellDictionary(Map> entries, AffixFile affix) { } this.prefixesByFirst = new HashMap<>(); this.prefixesWithoutMaterial = new ArrayList<>(); - for (final Affix prefix : prefixes) { + for (final Affix prefix : affix.prefixes) { final String material = prefix.affix(); if (material.isEmpty()) { prefixesWithoutMaterial.add(prefix); @@ -186,8 +186,11 @@ private HunspellDictionary(Map> entries, AffixFile affix) { */ public static HunspellDictionary load(Path affixFile, Path dictionaryFile) throws IOException { - if (affixFile == null || dictionaryFile == null) { - throw new IllegalArgumentException("affixFile and dictionaryFile must not be null"); + if (affixFile == null) { + throw new IllegalArgumentException("affixFile must not be null"); + } + if (dictionaryFile == null) { + throw new IllegalArgumentException("dictionaryFile must not be null"); } try (InputStream affix = Files.newInputStream(affixFile); InputStream dictionary = Files.newInputStream(dictionaryFile)) { @@ -208,14 +211,17 @@ public static HunspellDictionary load(Path affixFile, Path dictionaryFile) */ public static HunspellDictionary load(InputStream affixStream, InputStream dictionaryStream) throws IOException { - if (affixStream == null || dictionaryStream == null) { - throw new IllegalArgumentException("streams must not be null"); + if (affixStream == null) { + throw new IllegalArgumentException("affixStream must not be null"); + } + if (dictionaryStream == null) { + throw new IllegalArgumentException("dictionaryStream must not be null"); } - final byte[] affixBytes = readAll(affixStream); + final byte[] affixBytes = affixStream.readAllBytes(); final Charset charset = declaredCharset(affixBytes); final AffixFile affix = parseAffix(new String(affixBytes, charset)); final Map> entries = parseWordList( - new String(readAll(dictionaryStream), charset), affix.flagMode, + new String(dictionaryStream.readAllBytes(), charset), affix.flagMode, affix.flagAliases); return new HunspellDictionary(entries, affix); } @@ -230,18 +236,6 @@ List lookup(String word) { return entries.get(word); } - /** @return The prefix rules. */ - List prefixes() { - return prefixes; - } - - /** @return The suffix rules. */ - List suffixes() { - return suffixes; - } - - private static final List NO_AFFIXES = List.of(); - /** * The suffix rules whose affix material ends in the given character, which are the * only material-bearing rules that can be undone from a word ending in it. @@ -550,23 +544,6 @@ boolean circumfixOnly(Affix affix) { return circumfix != 0 && affix.allowsContinuation(circumfix); } - /** - * Reads a stream fully into memory. The stream is not closed. - * - * @param in The stream to drain. - * @return All bytes the stream produced. Never {@code null}. - * @throws IOException Thrown if reading fails. - */ - private static byte[] readAll(InputStream in) throws IOException { - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - final byte[] buffer = new byte[8192]; - int read; - while ((read = in.read(buffer)) >= 0) { - out.write(buffer, 0, read); - } - return out.toByteArray(); - } - /** * Finds the {@code SET} declaration by scanning the raw affix bytes as ASCII, which * is safe because the declaration itself is ASCII in every supported encoding. Both @@ -584,7 +561,7 @@ private static Charset declaredCharset(byte[] affixBytes) throws IOException { final String name = trim(trimmed.substring(4)); try { return Charset.forName(name); - } catch (RuntimeException e) { + } catch (IllegalCharsetNameException | UnsupportedCharsetException e) { throw new IOException("unsupported SET encoding: " + name, e); } } @@ -852,7 +829,13 @@ private static Map> parseWordList(String content, } } if (!flagAliases.isEmpty() && isCount(flagRun)) { - final int alias = Integer.parseInt(flagRun); + final int alias; + try { + alias = Integer.parseInt(flagRun); + } catch (NumberFormatException e) { + throw new IOException("malformed flag alias '" + flagRun + "' at line " + + (i + 1), e); + } if (alias < 1 || alias > flagAliases.size()) { throw new IOException("flag alias " + alias + " at line " + (i + 1) + " is outside the AF table of " + flagAliases.size() + " aliases"); diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 8703217ad8..1f42f36869 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Set; +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.stemmer.Stemmer; import opennlp.tools.stemmer.hunspell.HunspellDictionary.Affix; import opennlp.tools.stemmer.hunspell.HunspellDictionary.CompoundPosition; @@ -45,10 +46,19 @@ * implementation reads only the immutable dictionary state, so a single instance is * safe to share between threads.

* - * @since 3.0.0 */ +@ThreadSafe public class HunspellStemmer implements Stemmer { + /** + * The most part-licensing attempts one decomposition search may spend. Compounding + * searches every split of every tail, which on adversarial input with a + * one-character minimum part length grows without useful bound; the budget stops + * the search there, missing analyses rather than stalling, in line with the + * engine's fail-closed posture. + */ + private static final int PART_CHECK_BUDGET = 2048; + private final HunspellDictionary dictionary; /** @@ -93,7 +103,7 @@ public List stemAll(CharSequence word) { if (analyses.isEmpty()) { return List.of(surface); } - return List.copyOf(new ArrayList(analyses)); + return List.copyOf(analyses); } /** @@ -104,7 +114,7 @@ public List stemAll(CharSequence word) { * @param surface The surface form. * @return The variants in analysis order. Never {@code null} or empty. */ - private static List variants(String surface) { + private List variants(String surface) { final String lowered = StringUtil.toLowerCase(surface); return lowered.equals(surface) ? List.of(surface) : List.of(surface, lowered); } @@ -141,15 +151,6 @@ private void analyze(String word, Set analyses) { } } - /** - * The most part-licensing attempts one decomposition search may spend. Compounding - * searches every split of every tail, which on adversarial input with a - * one-character minimum part length grows without useful bound; the budget stops - * the search there, missing analyses rather than stalling, in line with the - * engine's fail-closed posture. - */ - private static final int PART_CHECK_BUDGET = 2048; - /** * Decomposes a word into listed compound parts when the affix analysis found * nothing: the first part must be admitted to open a compound, every further part @@ -417,7 +418,7 @@ private void collectPrefixedPartStem(String part, Affix prefix, * @param suffix Whether the rule is a suffix rule. * @return The candidate stem, or {@code null} when the rule does not apply. */ - private static String removeAffixInCompound(String part, Affix affix, boolean suffix) { + private String removeAffixInCompound(String part, Affix affix, boolean suffix) { if (affix.affix().isEmpty() && affix.strip().isEmpty()) { return affix.condition().matches(part) ? part : null; } @@ -561,7 +562,7 @@ private void undoCrossProductSuffix(String stem, Affix prefix, Affix suffix, * @param suffix The rule to undo. * @return The candidate stem, or {@code null} when the rule does not apply. */ - private static String removeSuffix(String word, Affix suffix) { + private String removeSuffix(String word, Affix suffix) { final String affix = suffix.affix(); final String strip = suffix.strip(); if (affix.isEmpty() && strip.isEmpty() || !word.endsWith(affix) @@ -583,7 +584,7 @@ private static String removeSuffix(String word, Affix suffix) { * @param prefix The rule to undo. * @return The candidate stem, or {@code null} when the rule does not apply. */ - private static String removePrefix(String word, Affix prefix) { + private String removePrefix(String word, Affix prefix) { final String affix = prefix.affix(); final String strip = prefix.strip(); if (affix.isEmpty() && strip.isEmpty() || !word.startsWith(affix) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java index 21d66be2b0..840c7a2c0b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -17,6 +17,7 @@ package opennlp.tools.stemmer.hunspell; +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.stemmer.Stemmer; import opennlp.tools.stemmer.StemmerFactory; @@ -26,8 +27,8 @@ * *

The factory is immutable and safe to share across threads.

* - * @since 3.0.0 */ +@ThreadSafe public class HunspellStemmerFactory implements StemmerFactory { private final HunspellDictionary dictionary; diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java index d6d12faa44..dcc7e997fc 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java @@ -75,7 +75,8 @@ void testEnglishInflections() throws IOException { void testGermanInflections() throws IOException { final HunspellStemmer stemmer = loadOrSkip("de_DE_frami"); Assertions.assertEquals("Kind", stemmer.stem("Kinder").toString()); - Assertions.assertEquals("Haus", stemmer.stem("Häuser").toString()); + // Haeuser, written with a-umlaut, stems to Haus + Assertions.assertEquals("Haus", stemmer.stem("H\u00E4user").toString()); Assertions.assertEquals("schnell", stemmer.stem("schnellsten").toString()); } @@ -84,7 +85,8 @@ void testGermanCompoundsDecompose() throws IOException { final HunspellStemmer stemmer = loadOrSkip("de_DE_frami"); // the exact part spellings follow the dictionary's own entries and may shift // between revisions; that ordinary compounds decompose at all must not - Assertions.assertTrue(stemmer.stemAll("Haustür").size() >= 2); + // Haustuer, written with u-umlaut, is Haus + Tuer + Assertions.assertTrue(stemmer.stemAll("Haust\u00FCr").size() >= 2); Assertions.assertTrue(stemmer.stemAll("Kinderzimmer").size() >= 2); Assertions.assertTrue(stemmer.stemAll("Abbildungsverzeichnis").size() >= 2); } @@ -92,8 +94,10 @@ void testGermanCompoundsDecompose() throws IOException { @Test void testHungarianInflections() throws IOException { final HunspellStemmer stemmer = loadOrSkip("hu_HU"); - Assertions.assertEquals("kutya", stemmer.stem("kutyák").toString()); + // kutyak, written with a-acute, is the plural of kutya + Assertions.assertEquals("kutya", stemmer.stem("kuty\u00E1k").toString()); Assertions.assertEquals("asztal", stemmer.stem("asztalon").toString()); - Assertions.assertEquals("könyv", stemmer.stem("könyveket").toString()); + // konyveket, written with o-umlaut, is an inflected form of konyv + Assertions.assertEquals("k\u00F6nyv", stemmer.stem("k\u00F6nyveket").toString()); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java index e19ce10729..5acbb793c5 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java @@ -167,13 +167,20 @@ void testFactorySharedAcrossThreads(@TempDir Path tempDir) throws Exception { } /** - * Verifies that the file-based entry point rejects {@code null} paths with the - * documented exception instead of failing later with an obscure error. + * Verifies that the file-based entry point rejects each {@code null} path with the + * documented exception naming the offending argument. + * + * @param tempDir A scratch directory managed by the test framework. */ @Test - void testNullPathsAreRejected() { - final IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, - () -> HunspellDictionary.load((Path) null, (Path) null)); - Assertions.assertEquals("affixFile and dictionaryFile must not be null", e.getMessage()); + void testNullPathsAreRejected(@TempDir Path tempDir) { + final Path present = tempDir.resolve("present.aff"); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load(null, present)); + Assertions.assertEquals("affixFile must not be null", e.getMessage()); + + e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load(present, null)); + Assertions.assertEquals("dictionaryFile must not be null", e.getMessage()); } } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index d4fe1a1961..f386ba5d9a 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -19,6 +19,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.List; @@ -532,8 +533,7 @@ void testMalformedInputFailsLoud() { new ByteArrayInputStream("SFX S 0 s [a\n".getBytes(StandardCharsets.UTF_8)), new ByteArrayInputStream("0\n".getBytes(StandardCharsets.UTF_8)))); Assertions.assertThrows(IllegalArgumentException.class, - () -> HunspellDictionary.load((java.io.InputStream) null, - (java.io.InputStream) null)); + () -> HunspellDictionary.load((InputStream) null, (InputStream) null)); Assertions.assertThrows(IllegalArgumentException.class, () -> new HunspellStemmer(null)); Assertions.assertThrows(IllegalArgumentException.class, @@ -629,10 +629,10 @@ void testSupplementaryFlagCharacterIsOneCodePointFlag() throws IOException { Assertions.assertTrue(HunspellDictionary.hasFlag(emoji.lookup("walk"), 0x1F600)); Assertions.assertFalse(HunspellDictionary.hasFlag(emoji.lookup("walk"), 0xD83D)); - final HunspellStemmer stemmer = new HunspellStemmer(load( + final HunspellStemmer supplementaryFlags = new HunspellStemmer(load( "FLAG UTF-8\nSFX \uD83D\uDE00 Y 1\nSFX \uD83D\uDE00 0 s .\n", "1\nwalk/\uD83D\uDE00\n")); - Assertions.assertEquals("walk", stemmer.stem("walks").toString()); + Assertions.assertEquals("walk", supplementaryFlags.stem("walks").toString()); } /** @@ -700,18 +700,24 @@ void testNumericDictionaryFlagsResolveThroughTheAliasTable() throws IOException /** * Verifies that an alias reference outside the AF table fails loud with the line - * and the table size, instead of silently flagging the entry with nothing. - * - * @throws IOException Thrown if the affix fixture fails to load. + * and the table size, instead of silently flagging the entry with nothing, and that + * a digit run too large for an alias number fails loud as well. */ @Test void testAliasReferenceOutsideTheTableFailsLoud() { - final IOException e = Assertions.assertThrows(IOException.class, () -> load( + IOException e = Assertions.assertThrows(IOException.class, () -> load( "AF 1\nAF S # 1\nSFX S Y 1\nSFX S 0 s .\n", "1\nwalk/2\n")); Assertions.assertEquals( "flag alias 2 at line 2 is outside the AF table of 1 aliases", e.getMessage()); + + e = Assertions.assertThrows(IOException.class, () -> load( + "AF 1\nAF S # 1\nSFX S Y 1\nSFX S 0 s .\n", + "1\nwalk/99999999999999999999\n")); + Assertions.assertEquals( + "malformed flag alias '99999999999999999999' at line 2", + e.getMessage()); } /** diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 8dc078d975..0c999099b9 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -73,11 +73,11 @@ new CachingStemmer(factory).stem("running"); // "run"]]>
Hunspell dictionaries - opennlp.tools.stemmer.hunspell is a clean-room engine over the - documented Hunspell dictionary format: a user-supplied + opennlp.tools.stemmer.hunspell implements the documented + Hunspell dictionary format: a user-supplied .aff affix file and its .dic word list. OpenNLP - bundles no dictionary data, so each dictionary's own license stays with - the files you download. The dictionary is immutable and safe to share; + bundles no dictionary data; dictionaries are downloaded separately, and + each states its own license. The dictionary is immutable and safe to share; HunspellStemmerFactory hands out a fresh stemmer per call. HunspellManualExampleTest asserts the behavior shown here. published one, and asserts the same stems for workers and worker. Acquisition helpers and the supported affix feature set live in - opennlp-core/opennlp-runtime/dev/README-hunspell-dictionaries.md. + dev/README-hunspell-dictionaries.md.
From e0b2b4d68fb21cda53e1f2f706cc1c55b4845431 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 24 Jul 2026 05:05:36 -0400 Subject: [PATCH 13/24] OPENNLP-1893: Add {@inheritDoc} to the stemmer overrides and trim empty javadoc lines --- .../tools/stemmer/hunspell/HunspellStemmer.java | 12 +++++++++++- .../stemmer/hunspell/HunspellStemmerFactory.java | 1 - 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 1f42f36869..aaf143881a 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -45,7 +45,6 @@ *

The {@link Stemmer} interface leaves thread safety to the implementation. This * implementation reads only the immutable dictionary state, so a single instance is * safe to share between threads.

- * */ @ThreadSafe public class HunspellStemmer implements Stemmer { @@ -74,12 +73,23 @@ public HunspellStemmer(HunspellDictionary dictionary) { this.dictionary = dictionary; } + /** + * {@inheritDoc} + * + *

Returns the first analysis, which prefers the word's own dictionary entry.

+ */ @Override public CharSequence stem(CharSequence word) { final List analyses = stemAll(word); return analyses.get(0); } + /** + * {@inheritDoc} + * + *

Returns every distinct analysis, or a single-element list of the unchanged word + * when it has none.

+ */ @Override public List stemAll(CharSequence word) { if (word == null) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java index 840c7a2c0b..1d6996c403 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -26,7 +26,6 @@ * {@link HunspellDictionary} and hands out {@link HunspellStemmer} instances over it. * *

The factory is immutable and safe to share across threads.

- * */ @ThreadSafe public class HunspellStemmerFactory implements StemmerFactory { From d8cc10e29e14a80970034aef5a9c8cb28c9f3041 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:01:04 -0400 Subject: [PATCH 14/24] OPENNLP-1893: Address review: fold the affix twins, extract tags, complete javadoc - Fold the two per-kind bucketing loops in the HunspellDictionary constructor into a single bucketByBoundary helper that takes the rule list, the kind, and the sink for the rules with empty affix material. - Fold collectSuffixedPartStem and collectPrefixedPartStem, which differed only in the boundary they face, into one collectAffixedPartStem with a suffix marker and an atEdge marker; document what atEdge means at each end. - Extract a parseValue helper for the single-integer directives so COMPOUNDMIN and COMPOUNDWORDMAX no longer share one case body that re-tests which directive it is. - Extract PREFIX_TAG, SUFFIX_TAG, and NO_MATERIAL constants and use them at the affix block header, the rule lines, and the strip and affix material checks. - Give FORBIDDENWORD its own case in the flag directive switch instead of letting the catch-all default assign it, and make that default throw for a directive listed on the outer switch but not handled on the inner one. - Add the missing javadoc on the AffixCondition and HunspellDictionary constructors, the Affix record components, and the splitLines, splitOn, and split helpers. - Convert the single-line accessor javadoc on the compounding and affix bucket getters to the {@return ...} form, and replace the hand-written prose on HunspellStemmerFactory.newStemmer with {@inheritDoc} plus the instancing note. - Trim commentary that restates the code: the bucketing rationale duplicated in HunspellStemmer, the LibreOffice Spanish anecdote on the code point flag reader, and the sentence left dangling in testGermanCompoundsDecompose. - Drop the defensive null and directory guards from the test helpers writeAndLoadFixture and load, which no caller can trip, and document what the real dictionary tests assert. - Fold the repeated ByteArrayInputStream plumbing in HunspellStemmerTest into two load overloads, one UTF-8 and one taking the charset the SET declaration test needs. - Turn the four table-style stemming tests into parameterized tests over their word and expected stem pairs, so a failing row names itself. - Add testNullArgumentsAreRejected, pinning the exact IllegalArgumentException message of every public entry point including the argument names the stream loader reports. - Correct the stemmer manual: name the example files after the fixture the test loads rather than en_US, and state that the printed stems are the fixture's, since which stem a published dictionary yields is that dictionary's decision. --- .../stemmer/hunspell/AffixCondition.java | 8 + .../stemmer/hunspell/HunspellDictionary.java | 177 ++++++++----- .../stemmer/hunspell/HunspellStemmer.java | 63 ++--- .../hunspell/HunspellStemmerFactory.java | 6 +- .../hunspell/HunspellRealDictionaryTest.java | 27 +- .../hunspell/HunspellStemmerFactoryTest.java | 7 +- .../stemmer/hunspell/HunspellStemmerTest.java | 240 +++++++++++------- opennlp-docs/src/docbkx/stemmer.xml | 11 +- 8 files changed, 324 insertions(+), 215 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java index a676b7e487..046913415f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java @@ -36,8 +36,16 @@ final class AffixCondition { private final char[][] accepted; /** Per position with a class: whether the class is negated; {@code null} rows unused. */ private final boolean[] negated; + /** Whether the owning rule is a suffix rule, which anchors the condition at the end. */ private final boolean suffix; + /** + * Initializes the condition. + * + * @param accepted The accepted characters per position. + * @param negated The negation marker per position. + * @param suffix Whether the owning rule is a suffix rule. + */ private AffixCondition(char[][] accepted, boolean[] negated, boolean suffix) { this.accepted = accepted; this.negated = negated; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 5970304a4e..cb94b3aa07 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -69,11 +69,14 @@ public final class HunspellDictionary { /** - * One parsed affix rule. {@code affix} is the surface material the rule adds to the - * stem, {@code strip} is the stem material the rule replaces (restored during - * analysis), {@code crossProduct} states whether the rule may combine with an affix - * of the opposite kind, and {@code continuation} lists the flags of further affixes - * that may stack on top of this one. + * One parsed affix rule of a {@code PFX} or {@code SFX} block. + * + * @param flag The flag naming the rule's block, which an entry carries to accept it. + * @param crossProduct Whether the rule may combine with an affix of the opposite kind. + * @param strip The stem material the rule replaces, restored during analysis. + * @param affix The surface material the rule adds to the stem. + * @param condition The condition the stem must satisfy for the rule to apply. + * @param continuation The flags of the further affixes that may stack on this one. */ record Affix(int flag, boolean crossProduct, String strip, String affix, AffixCondition condition, int[] continuation) { @@ -107,6 +110,15 @@ enum CompoundPosition { /** The shared empty bucket answered for characters no affix rule is keyed under. */ private static final List NO_AFFIXES = List.of(); + /** The line tag of a prefix block and of every rule line inside it. */ + private static final String PREFIX_TAG = "PFX"; + + /** The line tag of a suffix block and of every rule line inside it. */ + private static final String SUFFIX_TAG = "SFX"; + + /** The affix format's marker for absent strip or affix material. */ + private static final String NO_MATERIAL = "0"; + private final Map> entries; private final Map> suffixesByLast; private final List suffixesWithoutMaterial; @@ -128,6 +140,12 @@ enum CompoundPosition { private final boolean checkCompoundCase; private final boolean checkCompoundTriple; + /** + * Initializes the dictionary from the two parsed files. + * + * @param entries The words mapped to the flag sets of their entries. + * @param affix The parsed affix file. + */ private HunspellDictionary(Map> entries, AffixFile affix) { this.compoundFlag = affix.compoundFlag; this.compoundBegin = affix.compoundBegin; @@ -145,34 +163,39 @@ private HunspellDictionary(Map> entries, AffixFile affix) { this.checkCompoundCase = affix.checkCompoundCase; this.checkCompoundTriple = affix.checkCompoundTriple; this.entries = entries; - // Undoing a suffix requires the word to end with the rule's affix material, so - // only rules whose material ends in the word's last character can ever apply; - // the same holds for prefixes and the first character. Bucketing by that - // boundary character turns the per-word rule scan from the whole inventory into - // the one bucket plus the strip-only rules, whose empty material matches - // everywhere. - this.suffixesByLast = new HashMap<>(); + // A material-bearing rule can only be undone from a word whose boundary + // character matches its affix material, so bucketing by that character + // narrows each scan to one bucket plus the strip-only rules. this.suffixesWithoutMaterial = new ArrayList<>(); - for (final Affix suffix : affix.suffixes) { - final String material = suffix.affix(); - if (material.isEmpty()) { - suffixesWithoutMaterial.add(suffix); - } else { - suffixesByLast.computeIfAbsent(material.charAt(material.length() - 1), - key -> new ArrayList<>()).add(suffix); - } - } - this.prefixesByFirst = new HashMap<>(); + this.suffixesByLast = bucketByBoundary(affix.suffixes, true, suffixesWithoutMaterial); this.prefixesWithoutMaterial = new ArrayList<>(); - for (final Affix prefix : affix.prefixes) { - final String material = prefix.affix(); + this.prefixesByFirst = bucketByBoundary(affix.prefixes, false, prefixesWithoutMaterial); + } + + /** + * Buckets affix rules by the boundary character of their affix material, the last + * character for a suffix rule and the first for a prefix rule. + * + * @param rules The rules of one kind, in file order. + * @param suffix Whether the rules are suffix rules. + * @param withoutMaterial Collects the rules with empty affix material, which no + * boundary character keys. + * @return The rules keyed by their boundary character. Never {@code null}. + */ + private static Map> bucketByBoundary(List rules, + boolean suffix, List withoutMaterial) { + final Map> byBoundary = new HashMap<>(); + for (final Affix rule : rules) { + final String material = rule.affix(); if (material.isEmpty()) { - prefixesWithoutMaterial.add(prefix); + withoutMaterial.add(rule); } else { - prefixesByFirst.computeIfAbsent(material.charAt(0), - key -> new ArrayList<>()).add(prefix); + final char boundary = + suffix ? material.charAt(material.length() - 1) : material.charAt(0); + byBoundary.computeIfAbsent(boundary, key -> new ArrayList<>()).add(rule); } } + return byBoundary; } /** @@ -247,7 +270,7 @@ List suffixesEndingWith(char last) { return suffixesByLast.getOrDefault(last, NO_AFFIXES); } - /** @return The strip-only suffix rules, applicable to any word. Never {@code null}. */ + /** {@return the strip-only suffix rules, applicable to any word} Never {@code null}. */ List suffixesWithoutMaterial() { return suffixesWithoutMaterial; } @@ -263,38 +286,38 @@ List prefixesStartingWith(char first) { return prefixesByFirst.getOrDefault(first, NO_AFFIXES); } - /** @return The strip-only prefix rules, applicable to any word. Never {@code null}. */ + /** {@return the strip-only prefix rules, applicable to any word} Never {@code null}. */ List prefixesWithoutMaterial() { return prefixesWithoutMaterial; } - /** @return Whether the affix file declares any compounding flag at all. */ + /** {@return whether the affix file declares any compounding flag at all} */ boolean compoundsDeclared() { return compoundFlag != 0 || compoundBegin != 0 || compoundEnd != 0 || compoundMiddle != 0; } - /** @return The smallest length a compound part may have; at least {@code 1}. */ + /** {@return the smallest length a compound part may have} At least {@code 1}. */ int compoundMin() { return compoundMin; } - /** @return The largest number of parts a compound may have; {@code 0} is unbounded. */ + /** {@return the largest number of parts a compound may have} {@code 0} is unbounded. */ int compoundWordMax() { return compoundWordMax; } - /** @return Whether {@code CHECKCOMPOUNDDUP} forbids a part repeating its neighbor. */ + /** {@return whether {@code CHECKCOMPOUNDDUP} forbids a part repeating its neighbor} */ boolean checkCompoundDup() { return checkCompoundDup; } - /** @return Whether {@code CHECKCOMPOUNDCASE} forbids uppercase at part boundaries. */ + /** {@return whether {@code CHECKCOMPOUNDCASE} forbids uppercase at part boundaries} */ boolean checkCompoundCase() { return checkCompoundCase; } - /** @return Whether {@code CHECKCOMPOUNDTRIPLE} forbids triple letters at boundaries. */ + /** {@return whether {@code CHECKCOMPOUNDTRIPLE} forbids triple letters at boundaries} */ boolean checkCompoundTriple() { return checkCompoundTriple; } @@ -668,24 +691,18 @@ private static AffixFile parseAffix(String content) throws IOException { case "NEEDAFFIX", "PSEUDOROOT" -> result.needAffix = declared; case "ONLYINCOMPOUND" -> result.onlyInCompound = declared; case "CIRCUMFIX" -> result.circumfix = declared; - default -> result.forbiddenWord = declared; + case "FORBIDDENWORD" -> result.forbiddenWord = declared; + default -> throw new IOException( + "unhandled flag directive " + fields[0] + " at line " + (i + 1)); } i++; break; case "COMPOUNDMIN": + result.compoundMin = Math.max(1, parseValue(fields, i + 1)); + i++; + break; case "COMPOUNDWORDMAX": - if (fields.length < 2) { - throw new IOException(fields[0] + " line without a value at line " + (i + 1)); - } - try { - if ("COMPOUNDMIN".equals(fields[0])) { - result.compoundMin = Math.max(1, Integer.parseInt(fields[1])); - } else { - result.compoundWordMax = Math.max(0, Integer.parseInt(fields[1])); - } - } catch (NumberFormatException e) { - throw new IOException("malformed " + fields[0] + " at line " + (i + 1), e); - } + result.compoundWordMax = Math.max(0, parseValue(fields, i + 1)); i++; break; case "CHECKCOMPOUNDDUP": @@ -712,8 +729,8 @@ private static AffixFile parseAffix(String content) throws IOException { } i++; break; - case "PFX": - case "SFX": + case PREFIX_TAG: + case SUFFIX_TAG: i = parseAffixBlock(lines, i, fields, result); break; default: @@ -724,6 +741,25 @@ private static AffixFile parseAffix(String content) throws IOException { return result; } + /** + * Parses the integer value of a directive that carries exactly one. + * + * @param fields The already-split directive line. + * @param lineNumber The source line, for error messages. + * @return The parsed value. + * @throws IOException Thrown if the value is missing or is not an integer. + */ + private static int parseValue(String[] fields, int lineNumber) throws IOException { + if (fields.length < 2) { + throw new IOException(fields[0] + " line without a value at line " + lineNumber); + } + try { + return Integer.parseInt(fields[1]); + } catch (NumberFormatException e) { + throw new IOException("malformed " + fields[0] + " at line " + lineNumber, e); + } + } + /** * Parses one {@code PFX} or {@code SFX} block: the header line naming the flag, the * cross-product marker, and the rule count, followed by exactly that many rule @@ -741,7 +777,7 @@ private static int parseAffixBlock(String[] lines, int index, String[] header, if (header.length < 4) { throw new IOException("malformed affix header at line " + (index + 1)); } - final boolean suffix = "SFX".equals(header[0]); + final boolean suffix = SUFFIX_TAG.equals(header[0]); final int flag = parseFlag(header[1], result.flagMode, index + 1); final boolean crossProduct = "Y".equals(header[2]); final int count; @@ -759,7 +795,7 @@ private static int parseAffixBlock(String[] lines, int index, String[] header, if (fields.length < 5 || !fields[0].equals(header[0])) { throw new IOException("malformed affix rule at line " + (line + 1)); } - final String strip = "0".equals(fields[2]) ? "" : fields[2]; + final String strip = NO_MATERIAL.equals(fields[2]) ? "" : fields[2]; String affixText = fields[3]; int[] continuation = new int[0]; final int slash = affixText.indexOf('/'); @@ -767,7 +803,7 @@ private static int parseAffixBlock(String[] lines, int index, String[] header, continuation = parseFlags(affixText.substring(slash + 1), result.flagMode, line + 1); affixText = affixText.substring(0, slash); } - if ("0".equals(affixText)) { + if (NO_MATERIAL.equals(affixText)) { affixText = ""; } final Affix affix = new Affix(flag, crossProduct, strip, affixText, @@ -993,14 +1029,11 @@ private static int[] parseFlags(String text, FlagMode mode, int lineNumber) return flags; } default: { - // One flag per code point: published dictionaries, the Spanish one of the - // LibreOffice collection among them, name affix rules with supplementary - // characters under FLAG UTF-8, and reading per UTF-16 unit would split such - // a flag into two and reject the rule header as carrying two flags. A - // variation selector after a flag character selects its presentation, the - // emoji telephone against the text telephone, and is no flag of its own; the - // same collection writes such selectors, so they are dropped from flag - // identity. + // One flag per code point: published dictionaries name affix rules with + // supplementary characters under FLAG UTF-8, and reading per UTF-16 unit + // would split such a flag into a surrogate pair. A variation selector + // (U+FE00..U+FE0F) only selects a flag character's presentation and is + // dropped from flag identity. final int[] buffer = new int[text.codePointCount(0, text.length())]; int f = 0; for (int i = 0; i < text.length(); ) { @@ -1035,7 +1068,12 @@ private static int parseFlag(String text, FlagMode mode, int lineNumber) return flags[0]; } - /** Splits text into lines with a single character scan, tolerating CRLF endings. */ + /** + * Splits text into lines with a single character scan, tolerating CRLF endings. + * + * @param content The text to split. + * @return The lines without their terminators. Never {@code null}. + */ private static String[] splitLines(String content) { final List lines = new ArrayList<>(); int start = 0; @@ -1052,7 +1090,13 @@ private static String[] splitLines(String content) { return lines.toArray(new String[0]); } - /** Splits text on a separator character with a single character scan. */ + /** + * Splits text on a separator character with a single character scan. + * + * @param text The text to split. + * @param separator The separator character. + * @return The parts between the separators, empty ones included. Never {@code null}. + */ private static String[] splitOn(String text, char separator) { final List parts = new ArrayList<>(); int start = 0; @@ -1065,7 +1109,12 @@ private static String[] splitOn(String text, char separator) { return parts.toArray(new String[0]); } - /** Splits a line on whitespace with a single character scan. */ + /** + * Splits a line on whitespace with a single character scan. + * + * @param line The line to split. + * @return The whitespace-separated fields, without empty ones. Never {@code null}. + */ private static String[] split(String line) { final List parts = new ArrayList<>(); int start = -1; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index aaf143881a..bd9b230185 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -144,9 +144,6 @@ private void analyze(String word, Set analyses) { if (own != null && dictionary.validStandalone(own)) { analyses.add(word); } - // Only rules whose affix material ends in the word's last character can be - // undone from it, so each scan walks that bucket plus the strip-only rules - // instead of the whole inventory. for (final Affix suffix : dictionary.suffixesEndingWith(word.charAt(word.length() - 1))) { undoSuffix(word, suffix, analyses); } @@ -350,69 +347,45 @@ private void collectPartStems(String part, CompoundPosition position, stems.add(part); } for (final Affix suffix : dictionary.suffixesEndingWith(part.charAt(part.length() - 1))) { - collectSuffixedPartStem(part, suffix, position, last, stems); + collectAffixedPartStem(part, suffix, true, position, last, stems); } for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { - collectSuffixedPartStem(part, suffix, position, last, stems); + collectAffixedPartStem(part, suffix, true, position, last, stems); } for (final Affix prefix : dictionary.prefixesStartingWith(part.charAt(0))) { - collectPrefixedPartStem(part, prefix, position, first, stems); + collectAffixedPartStem(part, prefix, false, position, first, stems); } for (final Affix prefix : dictionary.prefixesWithoutMaterial()) { - collectPrefixedPartStem(part, prefix, position, first, stems); + collectAffixedPartStem(part, prefix, false, position, first, stems); } } /** - * Adds the stem of one suffixed part reading when the rule and the stem's entry - * admit it at the position. + * Adds the stem of one affixed part reading when the rule and the stem's entry admit + * it at the position. * * @param part The part spelling under analysis. - * @param suffix The suffix rule to undo. - * @param position The part's place in the compound. - * @param last Whether the part closes the word. - * @param stems The mutable, insertion-ordered set collecting the stems. - */ - private void collectSuffixedPartStem(String part, Affix suffix, - CompoundPosition position, boolean last, Set stems) { - if (dictionary.circumfixOnly(suffix) || dictionary.forbidsInCompound(suffix) - || (!last && !dictionary.permitsInside(suffix))) { - return; - } - final String stem = removeAffixInCompound(part, suffix, true); - if (stem == null) { - return; - } - final List flagSets = dictionary.lookup(stem); - if (flagSets != null && dictionary.supportsPart(flagSets, suffix.flag(), position, - dictionary.affixAdmits(suffix, position))) { - stems.add(stem); - } - } - - /** - * Adds the stem of one prefixed part reading when the rule and the stem's entry - * admit it at the position. - * - * @param part The part spelling under analysis. - * @param prefix The prefix rule to undo. + * @param affix The rule to undo. + * @param suffix Whether the rule is a suffix rule. * @param position The part's place in the compound. - * @param first Whether the part opens the word. + * @param atEdge Whether the part sits at the word end the rule faces, the closing part + * for a suffix rule and the opening part for a prefix rule; an affix + * facing another part instead needs the permit flag. * @param stems The mutable, insertion-ordered set collecting the stems. */ - private void collectPrefixedPartStem(String part, Affix prefix, - CompoundPosition position, boolean first, Set stems) { - if (dictionary.circumfixOnly(prefix) || dictionary.forbidsInCompound(prefix) - || (!first && !dictionary.permitsInside(prefix))) { + private void collectAffixedPartStem(String part, Affix affix, boolean suffix, + CompoundPosition position, boolean atEdge, Set stems) { + if (dictionary.circumfixOnly(affix) || dictionary.forbidsInCompound(affix) + || (!atEdge && !dictionary.permitsInside(affix))) { return; } - final String stem = removeAffixInCompound(part, prefix, false); + final String stem = removeAffixInCompound(part, affix, suffix); if (stem == null) { return; } final List flagSets = dictionary.lookup(stem); - if (flagSets != null && dictionary.supportsPart(flagSets, prefix.flag(), position, - dictionary.affixAdmits(prefix, position))) { + if (flagSets != null && dictionary.supportsPart(flagSets, affix.flag(), position, + dictionary.affixAdmits(affix, position))) { stems.add(stem); } } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java index 1d6996c403..a5e995752f 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -46,8 +46,10 @@ public HunspellStemmerFactory(HunspellDictionary dictionary) { } /** - * {@return a new {@link HunspellStemmer} over the shared dictionary} Every call - * creates a fresh instance; all instances read the same immutable dictionary. + * {@inheritDoc} + * + *

Every call creates a fresh {@link HunspellStemmer} over the same immutable + * dictionary.

*/ @Override public Stemmer newStemmer() { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java index dcc7e997fc..f1b0f7fe88 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellRealDictionaryTest.java @@ -59,6 +59,12 @@ private static HunspellStemmer loadOrSkip(String name) throws IOException { return new HunspellStemmer(HunspellDictionary.load(affix, words)); } + /** + * Checks everyday English inflections against {@code en_US}, plus the identity + * fallback on vocabulary no dictionary lists. + * + * @throws IOException Thrown if a present dictionary pair fails to load. + */ @Test void testEnglishInflections() throws IOException { final HunspellStemmer stemmer = loadOrSkip("en_US"); @@ -71,6 +77,12 @@ void testEnglishInflections() throws IOException { Assertions.assertEquals("zyzzyvax", stemmer.stem("zyzzyvax").toString()); } + /** + * Checks everyday German inflections against {@code de_DE_frami}: a plural, an + * umlauted plural, and a superlative. + * + * @throws IOException Thrown if a present dictionary pair fails to load. + */ @Test void testGermanInflections() throws IOException { final HunspellStemmer stemmer = loadOrSkip("de_DE_frami"); @@ -80,17 +92,28 @@ void testGermanInflections() throws IOException { Assertions.assertEquals("schnell", stemmer.stem("schnellsten").toString()); } + /** + * Checks that ordinary German compounds decompose against {@code de_DE_frami}. Only + * the part count is asserted: the exact part spellings follow the dictionary's own + * entries and may shift between its revisions. + * + * @throws IOException Thrown if a present dictionary pair fails to load. + */ @Test void testGermanCompoundsDecompose() throws IOException { final HunspellStemmer stemmer = loadOrSkip("de_DE_frami"); - // the exact part spellings follow the dictionary's own entries and may shift - // between revisions; that ordinary compounds decompose at all must not // Haustuer, written with u-umlaut, is Haus + Tuer Assertions.assertTrue(stemmer.stemAll("Haust\u00FCr").size() >= 2); Assertions.assertTrue(stemmer.stemAll("Kinderzimmer").size() >= 2); Assertions.assertTrue(stemmer.stemAll("Abbildungsverzeichnis").size() >= 2); } + /** + * Checks everyday Hungarian inflections against {@code hu_HU}: a plural and two + * case-suffixed forms. + * + * @throws IOException Thrown if a present dictionary pair fails to load. + */ @Test void testHungarianInflections() throws IOException { final HunspellStemmer stemmer = loadOrSkip("hu_HU"); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java index 5acbb793c5..fd599f57be 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactoryTest.java @@ -77,16 +77,11 @@ public class HunspellStemmerFactoryTest { * Writes the fixture dictionary pair into a directory and loads it through the * file-based {@link HunspellDictionary#load(Path, Path)} entry point. * - * @param directory The directory to write into. Must not be {@code null} and must - * denote an existing directory. + * @param directory The directory to write into. * @return The loaded dictionary. Never {@code null}. * @throws IOException Thrown if writing or loading fails. - * @throws IllegalArgumentException Thrown if {@code directory} is unusable. */ private static HunspellDictionary writeAndLoadFixture(Path directory) throws IOException { - if (directory == null || !Files.isDirectory(directory)) { - throw new IllegalArgumentException("directory must be an existing directory"); - } final Path affixFile = directory.resolve("fixture.aff"); final Path dictionaryFile = directory.resolve("fixture.dic"); Files.write(affixFile, AFFIX.getBytes(StandardCharsets.UTF_8)); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index f386ba5d9a..0929127904 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -27,6 +27,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import opennlp.tools.stemmer.Stemmer; @@ -67,51 +69,103 @@ public class HunspellStemmerTest { @BeforeAll static void loadDictionary() throws IOException { - final HunspellDictionary dictionary = HunspellDictionary.load( - new ByteArrayInputStream(AFFIX.getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(WORDS.getBytes(StandardCharsets.UTF_8))); - stemmer = new HunspellStemmer(dictionary); + stemmer = new HunspellStemmer(load(AFFIX, WORDS)); } - @Test - void testSuffixRules() { - Assertions.assertEquals("cat", stemmer.stem("cats").toString()); - Assertions.assertEquals("pony", stemmer.stem("ponies").toString()); - Assertions.assertEquals("box", stemmer.stem("boxes").toString()); - Assertions.assertEquals("make", stemmer.stem("making").toString()); - Assertions.assertEquals("lock", stemmer.stem("locking").toString()); + /** + * Loads a dictionary from in-memory affix and word-list content, both encoded as + * UTF-8, through the stream-based entry point. + * + * @param affix The {@code .aff} content. + * @param words The {@code .dic} content. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if the content is malformed. + */ + private static HunspellDictionary load(String affix, String words) throws IOException { + return load(affix, words, StandardCharsets.UTF_8); } - @Test - void testPrefixAndCrossProduct() { - Assertions.assertEquals("lock", stemmer.stem("unlock").toString()); - Assertions.assertEquals("lock", stemmer.stem("unlocks").toString()); - Assertions.assertEquals("lock", stemmer.stem("unlocking").toString()); + /** + * Loads a dictionary from in-memory affix and word-list content encoded in the given + * charset, through the stream-based entry point. + * + * @param affix The {@code .aff} content. + * @param words The {@code .dic} content. + * @param charset The charset both contents are encoded with. + * @return The loaded dictionary. Never {@code null}. + * @throws IOException Thrown if the content is malformed. + */ + private static HunspellDictionary load(String affix, String words, Charset charset) + throws IOException { + return HunspellDictionary.load(new ByteArrayInputStream(affix.getBytes(charset)), + new ByteArrayInputStream(words.getBytes(charset))); } - @Test - void testConditionsBlockWrongAnalyses() { - // the s rule requires a stem not ending in s, x, or y - Assertions.assertEquals("boxs", stemmer.stem("boxs").toString()); - // cat carries no G flag, so no ing analysis exists - Assertions.assertEquals("cating", stemmer.stem("cating").toString()); - // fish carries no flags at all - Assertions.assertEquals("fishs", stemmer.stem("fishs").toString()); + /** + * Verifies the fixture's suffix rules: the plural {@code -s}, the {@code y} to + * {@code ies} replacement, the {@code -es} plural after a sibilant, and the + * progressive {@code -ing} with and without the silent {@code e}. + * + * @param word The surface form to stem. + * @param expected The stem the fixture licenses. + */ + @ParameterizedTest + @CsvSource({"cats,cat", "ponies,pony", "boxes,box", "making,make", "locking,lock"}) + void testSuffixRules(String word, String expected) { + Assertions.assertEquals(expected, stemmer.stem(word).toString()); } - @Test - void testDirectLookupAndCase() { - Assertions.assertEquals("fish", stemmer.stem("fish").toString()); - Assertions.assertEquals("cat", stemmer.stem("Cats").toString()); - Assertions.assertEquals("lock", stemmer.stem("Unlocks").toString()); + /** + * Verifies prefix removal alone and combined with a suffix through the cross-product + * marker both rules declare. + * + * @param word The surface form to stem. + * @param expected The stem the fixture licenses. + */ + @ParameterizedTest + @CsvSource({"unlock,lock", "unlocks,lock", "unlocking,lock"}) + void testPrefixAndCrossProduct(String word, String expected) { + Assertions.assertEquals(expected, stemmer.stem(word).toString()); + } + + /** + * Verifies that an analysis a rule condition or a missing flag rejects is not + * reported: {@code boxs} fails the {@code [^sxy]} condition of the {@code -s} rule, + * {@code cat} carries no {@code G} flag, and {@code fish} carries no flag at all, so + * each surface form falls through unchanged. + * + * @param word The surface form to stem. + */ + @ParameterizedTest + @CsvSource({"boxs", "cating", "fishs"}) + void testConditionsBlockWrongAnalyses(String word) { + Assertions.assertEquals(word, stemmer.stem(word).toString()); } + /** + * Verifies that a listed word stems to itself and that a capitalized surface form is + * analyzed through its lowercase variant. + * + * @param word The surface form to stem. + * @param expected The stem the fixture licenses. + */ + @ParameterizedTest + @CsvSource({"fish,fish", "Cats,cat", "Unlocks,lock"}) + void testDirectLookupAndCase(String word, String expected) { + Assertions.assertEquals(expected, stemmer.stem(word).toString()); + } + + /** Verifies that a word with no analysis is returned unchanged as its only analysis. */ @Test void testUnknownWordsPassThroughUnchanged() { Assertions.assertEquals("zebras", stemmer.stem("zebras").toString()); Assertions.assertEquals(1, stemmer.stemAll("zebras").size()); } + /** + * Verifies that {@link HunspellStemmer#stemAll(CharSequence)} reports the analyses + * and that {@link HunspellStemmer#stem(CharSequence)} answers the first of them. + */ @Test void testStemAllReportsEveryAnalysis() { Assertions.assertEquals(1, stemmer.stemAll("unlocks").size()); @@ -120,20 +174,22 @@ void testStemAllReportsEveryAnalysis() { Assertions.assertEquals("lock", stemmer.stemAll("lock").get(0).toString()); } + /** + * Verifies twofold suffix removal: the plural {@code -s} stacks on the comparative + * {@code -er} through the continuation class the outer rule declares, while the inner + * flag alone licenses nothing because no entry carries it. + * + * @throws IOException Thrown if the fixture fails to load. + */ @Test void testTwofoldSuffixesThroughContinuationClasses() throws IOException { - final String affix = String.join("\n", + final HunspellStemmer twofold = new HunspellStemmer(load(String.join("\n", "SET UTF-8", "SFX A Y 1", "SFX A 0 er/B .", "SFX B Y 1", "SFX B 0 s .", - ""); - final String words = String.join("\n", "1", "kind/A", ""); - final HunspellDictionary dictionary = HunspellDictionary.load( - new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); - final HunspellStemmer twofold = new HunspellStemmer(dictionary); + ""), String.join("\n", "1", "kind/A", ""))); Assertions.assertEquals("kind", twofold.stem("kinder").toString()); Assertions.assertEquals("kind", twofold.stem("kinders").toString()); @@ -141,64 +197,51 @@ void testTwofoldSuffixesThroughContinuationClasses() throws IOException { Assertions.assertEquals("kinds", twofold.stem("kinds").toString()); } + /** + * Verifies {@code FLAG num} mode: a comma-separated run of decimal numbers is the + * entry's flag set, and an affix block named by one of them applies. + * + * @throws IOException Thrown if the fixture fails to load. + */ @Test void testNumericFlagMode() throws IOException { - final String affix = String.join("\n", + final HunspellDictionary dictionary = load(String.join("\n", "SET UTF-8", "FLAG num", "SFX 100 Y 1", "SFX 100 0 s .", - ""); - final String words = String.join("\n", "1", "walk/100,7", ""); - final HunspellDictionary dictionary = HunspellDictionary.load( - new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + ""), String.join("\n", "1", "walk/100,7", "")); Assertions.assertEquals("walk", new HunspellStemmer(dictionary).stem("walks").toString()); } + /** + * Verifies {@code FLAG long} mode: each pair of characters in the run is one flag, + * and an affix block named by such a pair applies. + * + * @throws IOException Thrown if the fixture fails to load. + */ @Test void testLongFlagMode() throws IOException { - final String affix = String.join("\n", + final HunspellDictionary dictionary = load(String.join("\n", "SET UTF-8", "FLAG long", "SFX Aa Y 1", "SFX Aa 0 s .", - ""); - final String words = String.join("\n", "1", "walk/AaBb", ""); - final HunspellDictionary dictionary = HunspellDictionary.load( - new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + ""), String.join("\n", "1", "walk/AaBb", "")); Assertions.assertEquals("walk", new HunspellStemmer(dictionary).stem("walks").toString()); } - @Test - void testFactoryHandsOutWorkingStemmers() throws IOException { - final HunspellDictionary dictionary = HunspellDictionary.load( - new ByteArrayInputStream(AFFIX.getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(WORDS.getBytes(StandardCharsets.UTF_8))); - final Stemmer fresh = new HunspellStemmerFactory(dictionary).newStemmer(); - Assertions.assertEquals("pony", fresh.stem("ponies").toString()); - } - /** - * Loads a dictionary from in-memory affix and word-list content, both encoded as - * UTF-8, through the stream-based entry point. + * Verifies that a stemmer minted by the factory analyzes against the same dictionary. * - * @param affix The {@code .aff} content. Must not be {@code null}. - * @param words The {@code .dic} content. Must not be {@code null}. - * @return The loaded dictionary. Never {@code null}. - * @throws IOException Thrown if the content is malformed. - * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + * @throws IOException Thrown if the fixture fails to load. */ - private static HunspellDictionary load(String affix, String words) throws IOException { - if (affix == null || words == null) { - throw new IllegalArgumentException("affix and words must not be null"); - } - return HunspellDictionary.load( - new ByteArrayInputStream(affix.getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(words.getBytes(StandardCharsets.UTF_8))); + @Test + void testFactoryHandsOutWorkingStemmers() throws IOException { + final Stemmer fresh = new HunspellStemmerFactory(load(AFFIX, WORDS)).newStemmer(); + Assertions.assertEquals("pony", fresh.stem("ponies").toString()); } /** @@ -255,15 +298,11 @@ void testPositiveCharacterClassRejectsCandidate() { @Test void testSetDeclarationSelectsEncoding() throws IOException { final Charset latin1 = StandardCharsets.ISO_8859_1; - final String affix = String.join("\n", + final HunspellDictionary dictionary = load(String.join("\n", "SET ISO8859-1", "SFX S Y 1", "SFX S 0 s .", - ""); - final String words = "1\ncaf\u00E9/S\n"; - final HunspellDictionary dictionary = HunspellDictionary.load( - new ByteArrayInputStream(affix.getBytes(latin1)), - new ByteArrayInputStream(words.getBytes(latin1))); + ""), "1\ncaf\u00E9/S\n", latin1); final HunspellStemmer latin1Stemmer = new HunspellStemmer(dictionary); Assertions.assertEquals("caf\u00E9", latin1Stemmer.stem("caf\u00E9s").toString()); Assertions.assertEquals("caf\u00E9", latin1Stemmer.stem("caf\u00E9").toString()); @@ -521,24 +560,43 @@ void testEmptyFlagRunYieldsNoFlagsInEveryMode() throws IOException { load("FLAG num\n", "1\nword/\n").lookup("word").get(0).length); } + /** Verifies that a malformed affix file aborts the load instead of loading partially. */ @Test void testMalformedInputFailsLoud() { - Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( - new ByteArrayInputStream("SFX S Y 2\nSFX S 0 s .\n".getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream("1\ncat/S\n".getBytes(StandardCharsets.UTF_8)))); - Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( - new ByteArrayInputStream("SET NO-SUCH-ENCODING\n".getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream("0\n".getBytes(StandardCharsets.UTF_8)))); - Assertions.assertThrows(IOException.class, () -> HunspellDictionary.load( - new ByteArrayInputStream("SFX S 0 s [a\n".getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream("0\n".getBytes(StandardCharsets.UTF_8)))); - Assertions.assertThrows(IllegalArgumentException.class, - () -> HunspellDictionary.load((InputStream) null, (InputStream) null)); - Assertions.assertThrows(IllegalArgumentException.class, + Assertions.assertThrows(IOException.class, + () -> load("SFX S Y 2\nSFX S 0 s .\n", "1\ncat/S\n")); + Assertions.assertThrows(IOException.class, + () -> load("SET NO-SUCH-ENCODING\n", "0\n")); + Assertions.assertThrows(IOException.class, () -> load("SFX S 0 s [a\n", "0\n")); + } + + /** + * Verifies that every entry point rejects a {@code null} argument with the documented + * exception, and that the stream-based loader names the offending argument the way + * its file-based sibling does. + */ + @Test + void testNullArgumentsAreRejected() { + final InputStream present = new ByteArrayInputStream(new byte[0]); + IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load(null, present)); + Assertions.assertEquals("affixStream must not be null", e.getMessage()); + + e = Assertions.assertThrows(IllegalArgumentException.class, + () -> HunspellDictionary.load(present, null)); + Assertions.assertEquals("dictionaryStream must not be null", e.getMessage()); + + e = Assertions.assertThrows(IllegalArgumentException.class, () -> new HunspellStemmer(null)); - Assertions.assertThrows(IllegalArgumentException.class, + Assertions.assertEquals("dictionary must not be null", e.getMessage()); + + e = Assertions.assertThrows(IllegalArgumentException.class, () -> new HunspellStemmerFactory(null)); - Assertions.assertThrows(IllegalArgumentException.class, () -> stemmer.stemAll(null)); + Assertions.assertEquals("dictionary must not be null", e.getMessage()); + + e = Assertions.assertThrows(IllegalArgumentException.class, + () -> stemmer.stemAll(null)); + Assertions.assertEquals("word must not be null", e.getMessage()); } /** diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 0c999099b9..55970b6851 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -82,16 +82,17 @@ new CachingStemmer(factory).stem("running"); // "run"]]> HunspellManualExampleTest asserts the behavior shown here. - The in-tree test uses a project-authored miniature dictionary instead of a - published one, and asserts the same stems for workers and - worker. Acquisition helpers and the supported affix feature - set live in + The stems above are those of the project-authored miniature dictionary the + test loads, which lists work with an agentive and a plural + suffix; the test asserts the same stem for worker. Which stem a + published dictionary yields for a given form is decided by that dictionary. + Acquisition helpers and the supported affix feature set live in dev/README-hunspell-dictionaries.md. From ba8ab36bbe30e80d2d31098464aa9d8554fd1093 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 07:00:37 -0400 Subject: [PATCH 15/24] OPENNLP-1893: Fail loud on result-altering unsupported affix directives Reject ICONV, OCONV, and COMPLEXPREFIXES at load time; keep skipping cosmetic tables such as REP. Copy lookup results defensively and document the compound search budget on HunspellStemmer. --- dev/README-hunspell-dictionaries.md | 2 +- .../stemmer/hunspell/HunspellDictionary.java | 30 ++++++++++++++----- .../stemmer/hunspell/HunspellStemmer.java | 10 ++++--- .../stemmer/hunspell/HunspellStemmerTest.java | 30 +++++++++++++++++++ opennlp-docs/src/docbkx/stemmer.xml | 3 ++ 5 files changed, 62 insertions(+), 13 deletions(-) diff --git a/dev/README-hunspell-dictionaries.md b/dev/README-hunspell-dictionaries.md index 34cb5c2783..76175be339 100644 --- a/dev/README-hunspell-dictionaries.md +++ b/dev/README-hunspell-dictionaries.md @@ -61,4 +61,4 @@ The in-tree tests run against project-authored fixtures only. An opt-in test cla ## What the engine supports -Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, the `AF` flag alias table, the `SET` encoding declaration, compound decomposition under `COMPOUNDFLAG`, the positional `COMPOUNDBEGIN`/`COMPOUNDMIDDLE`/`COMPOUNDEND` flags, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, and the `CHECKCOMPOUNDDUP`/`CHECKCOMPOUNDCASE`/`CHECKCOMPOUNDTRIPLE` declarations (compound parts stand on their entries alone or on an entry plus one affix, the zero and dash suffixes dictionaries position linking forms with included), the blocking flags `NEEDAFFIX` (alias `PSEUDOROOT`), `ONLYINCOMPOUND`, and `FORBIDDENWORD`, which keep virtual stems, compound-only parts, and forbidden words out of the reported analyses, and `CIRCUMFIX`, which binds marked prefix and suffix halves to one another as in the German `ge...t` participle. Conversion tables and the remaining compound machinery are not interpreted; rules that use them simply do not fire, so unsupported analyses are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, the `AF` flag alias table, the `SET` encoding declaration, compound decomposition under `COMPOUNDFLAG`, the positional `COMPOUNDBEGIN`/`COMPOUNDMIDDLE`/`COMPOUNDEND` flags, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, and the `CHECKCOMPOUNDDUP`/`CHECKCOMPOUNDCASE`/`CHECKCOMPOUNDTRIPLE` declarations (compound parts stand on their entries alone or on an entry plus one affix, the zero and dash suffixes dictionaries position linking forms with included), the blocking flags `NEEDAFFIX` (alias `PSEUDOROOT`), `ONLYINCOMPOUND`, and `FORBIDDENWORD`, which keep virtual stems, compound-only parts, and forbidden words out of the reported analyses, and `CIRCUMFIX`, which binds marked prefix and suffix halves to one another as in the German `ge...t` participle. Directives that would change stems when ignored (`ICONV`, `OCONV`, `COMPLEXPREFIXES`) fail at load time. Cosmetic tables such as `REP`, `MAP`, and `KEY` are skipped, so analyses that would need them are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index cb94b3aa07..7c57022ef3 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -56,9 +56,10 @@ * {@code ONLYINCOMPOUND}, and {@code FORBIDDENWORD}, which suppress analyses the * dictionary marks as virtual stems, compound-only parts, or forbidden words; and * {@code CIRCUMFIX}, which binds marked prefix and suffix halves to one another. - * Conversion tables and the remaining compound machinery are not interpreted in this - * version; rules using them simply do not fire, so unsupported analyses are missed - * rather than invented.

+ * Directives that would change stems when ignored ({@code ICONV}, {@code OCONV}, + * {@code COMPLEXPREFIXES}) are rejected at load time. Cosmetic tables such as + * {@code REP}, {@code MAP}, and {@code KEY} are skipped, so analyses that would need + * them are missed rather than invented.

* *

Instances are immutable and safe to share between threads.

* @@ -256,7 +257,15 @@ public static HunspellDictionary load(InputStream affixStream, * @return The flag sets of all matching entries, or {@code null} when absent. */ List lookup(String word) { - return entries.get(word); + final List found = entries.get(word); + if (found == null) { + return null; + } + final List copy = new ArrayList<>(found.size()); + for (final int[] flags : found) { + copy.add(flags.clone()); + } + return copy; } /** @@ -633,13 +642,13 @@ private static final class AffixFile { /** * Parses the affix file: the {@code FLAG} declaration, the {@code AF} flag alias * table, the compound and blocking flag declarations, and the {@code PFX} and - * {@code SFX} blocks. Directives outside the supported set (conversion tables, - * suggestion options, the remaining compound machinery, ...) are skipped, so their - * rules never fire and unsupported analyses are missed rather than invented. + * {@code SFX} blocks. Result-altering unsupported directives fail loud; + * cosmetic ones are skipped. * * @param content The decoded affix file content. * @return The parsed rules and flag mode. Never {@code null}. - * @throws IOException Thrown if a supported directive is malformed. + * @throws IOException Thrown if a supported directive is malformed, or if + * {@code ICONV}, {@code OCONV}, or {@code COMPLEXPREFIXES} appears. */ private static AffixFile parseAffix(String content) throws IOException { final AffixFile result = new AffixFile(); @@ -733,6 +742,11 @@ private static AffixFile parseAffix(String content) throws IOException { case SUFFIX_TAG: i = parseAffixBlock(lines, i, fields, result); break; + case "ICONV": + case "OCONV": + case "COMPLEXPREFIXES": + throw new IOException("unsupported affix directive '" + fields[0] + + "' at line " + (i + 1)); default: i++; break; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index bd9b230185..996b2f5c08 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -42,12 +42,14 @@ * parts ({@code ONLYINCOMPOUND}), or forbidden words ({@code FORBIDDENWORD}) never * count as standalone analyses, matching how hunspell reads those flags.

* - *

The {@link Stemmer} interface leaves thread safety to the implementation. This - * implementation reads only the immutable dictionary state, so a single instance is - * safe to share between threads.

+ *

Compound part search is capped at 2048 part-licensing attempts per input word; + * beyond that budget further compound analyses are skipped. The {@link Stemmer} + * interface leaves thread safety to the implementation. This implementation reads only + * the immutable dictionary state, so a single instance is safe to share between + * threads.

*/ @ThreadSafe -public class HunspellStemmer implements Stemmer { +public final class HunspellStemmer implements Stemmer { /** * The most part-licensing attempts one decomposition search may spend. Compounding diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 0929127904..371e6ff576 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -1218,4 +1218,34 @@ void testForbiddenEntryBlocksItsDecomposition() throws IOException { Assertions.assertEquals(List.of("doghouse"), stemmer.stemAll("doghouse")); Assertions.assertEquals(List.of("cat", "house"), stemmer.stemAll("cathouse")); } + + /** + * Verifies that result-altering unsupported affix directives fail at load time. + * Ignoring {@code ICONV}, {@code OCONV}, or {@code COMPLEXPREFIXES} would change + * stems with no signal. + */ + @ParameterizedTest + @CsvSource({ + "ICONV, ICONV 1", + "OCONV, OCONV 1", + "COMPLEXPREFIXES, COMPLEXPREFIXES" + }) + void testResultAlteringUnsupportedDirectiveFailsLoud(String name, String line) { + final IOException e = Assertions.assertThrows(IOException.class, + () -> load(line + "\n", "0\n")); + Assertions.assertEquals("unsupported affix directive '" + name + "' at line 1", + e.getMessage()); + } + + /** + * Verifies that a cosmetic unsupported directive such as {@code REP} is skipped so + * the dictionary still loads. + * + * @throws IOException Thrown if the fixture fails to load. + */ + @Test + void testCosmeticUnsupportedDirectiveIsSkipped() throws IOException { + final HunspellDictionary dictionary = load("REP 1\nREP alot a lot\n", "1\nlock\n"); + Assertions.assertNotNull(dictionary.lookup("lock")); + } } diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 55970b6851..96dd33ecdd 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -94,6 +94,9 @@ stemmer.stem("table"); // "table" (unknown vocabulary is unchanged)]]> published dictionary yields for a given form is decided by that dictionary. Acquisition helpers and the supported affix feature set live in dev/README-hunspell-dictionaries.md. + Directives that would change stems when ignored + (ICONV, OCONV, COMPLEXPREFIXES) + fail at load time; cosmetic tables such as REP are skipped. From 8cede5e29d9da400ed7ea3534f882a29b15c33fd Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 08:30:51 -0400 Subject: [PATCH 16/24] OPENNLP-1893: Bound stream size and match affix conditions by code point Reject affix and dictionary streams above MAX_STREAM_BYTES. Affix conditions and boundary bucketing use Unicode code points so supplementary characters agree with FLAG UTF-8. Document the ceiling in the stemmer chapter and pin both behaviors in tests. --- .../stemmer/hunspell/AffixCondition.java | 72 +++++++---- .../stemmer/hunspell/HunspellDictionary.java | 83 +++++++++---- .../stemmer/hunspell/HunspellStemmer.java | 28 +++-- .../hunspell/HunspellStemmerFactory.java | 2 + .../stemmer/hunspell/HunspellStemmerTest.java | 117 ++++++++++++++++++ opennlp-docs/src/docbkx/stemmer.xml | 2 + 6 files changed, 247 insertions(+), 57 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java index 046913415f..19ce657ccc 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/AffixCondition.java @@ -22,18 +22,19 @@ import java.util.List; /** - * One parsed affix condition: a fixed-length sequence of literal characters and + * One parsed affix condition: a fixed-length sequence of literal code points and * bracketed character classes, matched with a single scan and no regular expressions. * A suffix condition anchors at the end of the candidate stem, a prefix condition at - * its start; the condition {@code .} matches everything. + * its start; the condition {@code .} matches everything. Positions are Unicode code + * points so supplementary characters agree with {@code FLAG UTF-8} flag reading. */ final class AffixCondition { /** The shared instance for the condition {@code .}, which accepts every stem. */ - private static final AffixCondition ANY = new AffixCondition(new char[0][], null, true); + private static final AffixCondition ANY = new AffixCondition(new int[0][], null, true); - /** Per position: the accepted characters, or {@code null} for any character. */ - private final char[][] accepted; + /** Per position: the accepted code points, or {@code null} for any code point. */ + private final int[][] accepted; /** Per position with a class: whether the class is negated; {@code null} rows unused. */ private final boolean[] negated; /** Whether the owning rule is a suffix rule, which anchors the condition at the end. */ @@ -42,20 +43,20 @@ final class AffixCondition { /** * Initializes the condition. * - * @param accepted The accepted characters per position. + * @param accepted The accepted code points per position. * @param negated The negation marker per position. * @param suffix Whether the owning rule is a suffix rule. */ - private AffixCondition(char[][] accepted, boolean[] negated, boolean suffix) { + private AffixCondition(int[][] accepted, boolean[] negated, boolean suffix) { this.accepted = accepted; this.negated = negated; this.suffix = suffix; } /** - * Parses a condition field. Each pattern position is a literal character, a - * {@code .} matching any character, or a bracketed class such as {@code [sx]}; a - * class starting with {@code ^} is negated and matches any character outside it. + * Parses a condition field. Each pattern position is a literal code point, a + * {@code .} matching any code point, or a bracketed class such as {@code [sx]}; a + * class starting with {@code ^} is negated and matches any code point outside it. * * @param pattern The condition text from the affix rule. * @param suffix Whether the owning rule is a suffix rule. @@ -68,12 +69,12 @@ static AffixCondition parse(String pattern, boolean suffix, int lineNumber) if (".".equals(pattern)) { return ANY; } - final List positions = new ArrayList<>(); + final List positions = new ArrayList<>(); final List negations = new ArrayList<>(); int i = 0; while (i < pattern.length()) { - final char c = pattern.charAt(i); - if (c == '[') { + final int codePoint = pattern.codePointAt(i); + if (codePoint == '[') { final int end = pattern.indexOf(']', i + 1); if (end < 0) { throw new IOException("unterminated character class at line " + lineNumber); @@ -84,20 +85,20 @@ static AffixCondition parse(String pattern, boolean suffix, int lineNumber) negate = true; members = members.substring(1); } - positions.add(members.toCharArray()); + positions.add(toCodePoints(members)); negations.add(negate); i = end + 1; - } else if (c == '.') { + } else if (codePoint == '.') { positions.add(null); negations.add(false); i++; } else { - positions.add(new char[] {c}); + positions.add(new int[] {codePoint}); negations.add(false); - i++; + i += Character.charCount(codePoint); } } - final char[][] accepted = positions.toArray(new char[0][]); + final int[][] accepted = positions.toArray(new int[0][]); final boolean[] negated = new boolean[accepted.length]; for (int p = 0; p < negated.length; p++) { negated[p] = negations.get(p); @@ -105,10 +106,29 @@ static AffixCondition parse(String pattern, boolean suffix, int lineNumber) return new AffixCondition(accepted, negated, suffix); } + /** + * Collects the code points of a character-class body. + * + * @param members The class body text. + * @return The code points in order. Never {@code null}. + */ + private static int[] toCodePoints(String members) { + final int[] codePoints = new int[members.codePointCount(0, members.length())]; + int i = 0; + int out = 0; + while (i < members.length()) { + final int codePoint = members.codePointAt(i); + codePoints[out++] = codePoint; + i += Character.charCount(codePoint); + } + return codePoints; + } + /** * Tests a candidate stem against the condition at its anchored side: the last * positions of the stem for a suffix condition, the first positions for a prefix - * condition. A stem shorter than the condition never matches. + * condition. A stem shorter than the condition never matches. Length is in code + * points. * * @param stem The candidate stem after affix removal and strip restoration. * @return {@code true} if the stem satisfies the condition. @@ -117,19 +137,21 @@ boolean matches(String stem) { if (accepted.length == 0) { return true; } - if (stem.length() < accepted.length) { + final int stemPoints = stem.codePointCount(0, stem.length()); + if (stemPoints < accepted.length) { return false; } - final int offset = suffix ? stem.length() - accepted.length : 0; + int offset = suffix ? stem.offsetByCodePoints(0, stemPoints - accepted.length) : 0; for (int p = 0; p < accepted.length; p++) { - final char[] members = accepted[p]; + final int[] members = accepted[p]; + final int codePoint = stem.codePointAt(offset); + offset += Character.charCount(codePoint); if (members == null) { continue; } - final char c = stem.charAt(offset + p); boolean member = false; - for (final char candidate : members) { - if (candidate == c) { + for (final int candidate : members) { + if (candidate == codePoint) { member = true; break; } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 7c57022ef3..2c3123a952 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -65,10 +65,18 @@ * * @see HunspellStemmer * @see HunspellStemmerFactory + * @since 3.0.0 */ @ThreadSafe public final class HunspellDictionary { + /** + * Inclusive upper bound on bytes buffered from one affix or dictionary stream + * during {@link #load(InputStream, InputStream)}. Larger streams fail with + * {@link IOException}. + */ + static final int MAX_STREAM_BYTES = 64 * 1024 * 1024; + /** * One parsed affix rule of a {@code PFX} or {@code SFX} block. * @@ -121,9 +129,9 @@ enum CompoundPosition { private static final String NO_MATERIAL = "0"; private final Map> entries; - private final Map> suffixesByLast; + private final Map> suffixesByLast; private final List suffixesWithoutMaterial; - private final Map> prefixesByFirst; + private final Map> prefixesByFirst; private final List prefixesWithoutMaterial; private final int compoundFlag; private final int compoundBegin; @@ -174,25 +182,26 @@ private HunspellDictionary(Map> entries, AffixFile affix) { } /** - * Buckets affix rules by the boundary character of their affix material, the last - * character for a suffix rule and the first for a prefix rule. + * Buckets affix rules by the boundary code point of their affix material, the last + * code point for a suffix rule and the first for a prefix rule. * * @param rules The rules of one kind, in file order. * @param suffix Whether the rules are suffix rules. * @param withoutMaterial Collects the rules with empty affix material, which no - * boundary character keys. - * @return The rules keyed by their boundary character. Never {@code null}. + * boundary code point keys. + * @return The rules keyed by their boundary code point. Never {@code null}. */ - private static Map> bucketByBoundary(List rules, + private static Map> bucketByBoundary(List rules, boolean suffix, List withoutMaterial) { - final Map> byBoundary = new HashMap<>(); + final Map> byBoundary = new HashMap<>(); for (final Affix rule : rules) { final String material = rule.affix(); if (material.isEmpty()) { withoutMaterial.add(rule); } else { - final char boundary = - suffix ? material.charAt(material.length() - 1) : material.charAt(0); + final int boundary = suffix + ? material.codePointBefore(material.length()) + : material.codePointAt(0); byBoundary.computeIfAbsent(boundary, key -> new ArrayList<>()).add(rule); } } @@ -223,14 +232,16 @@ public static HunspellDictionary load(Path affixFile, Path dictionaryFile) } /** - * Loads a dictionary from its two streams. + * Loads a dictionary from its two streams. Each stream is buffered up to + * {@link #MAX_STREAM_BYTES} bytes; a larger stream fails with {@link IOException}. * * @param affixStream The {@code .aff} affix content. Must not be {@code null}. Not * closed. * @param dictionaryStream The {@code .dic} word list content. Must not be * {@code null}. Not closed. * @return The loaded dictionary. Never {@code null}. - * @throws IOException Thrown if reading fails or the content is malformed. + * @throws IOException Thrown if reading fails, a stream exceeds + * {@link #MAX_STREAM_BYTES}, or the content is malformed. * @throws IllegalArgumentException Thrown if a parameter is {@code null}. */ public static HunspellDictionary load(InputStream affixStream, @@ -241,15 +252,45 @@ public static HunspellDictionary load(InputStream affixStream, if (dictionaryStream == null) { throw new IllegalArgumentException("dictionaryStream must not be null"); } - final byte[] affixBytes = affixStream.readAllBytes(); + final byte[] affixBytes = readBounded(affixStream, MAX_STREAM_BYTES, "affix stream"); final Charset charset = declaredCharset(affixBytes); final AffixFile affix = parseAffix(new String(affixBytes, charset)); final Map> entries = parseWordList( - new String(dictionaryStream.readAllBytes(), charset), affix.flagMode, - affix.flagAliases); + new String(readBounded(dictionaryStream, MAX_STREAM_BYTES, "dictionary stream"), + charset), + affix.flagMode, affix.flagAliases); return new HunspellDictionary(entries, affix); } + /** + * Reads an input stream into a byte array, failing when more than {@code maxBytes} + * arrive. + * + * @param in The stream to read. Not closed. + * @param maxBytes The inclusive upper bound on buffered bytes. + * @param label The stream name used in the error message. + * @return The buffered bytes. Never {@code null}. + * @throws IOException Thrown if reading fails or the stream exceeds {@code maxBytes}. + */ + static byte[] readBounded(InputStream in, int maxBytes, String label) + throws IOException { + final byte[] chunk = new byte[8192]; + byte[] buffer = new byte[Math.min(8192, maxBytes)]; + int size = 0; + int n; + while ((n = in.read(chunk)) >= 0) { + if (size + n > maxBytes) { + throw new IOException(label + " size exceeds safe limit of " + maxBytes); + } + if (size + n > buffer.length) { + buffer = Arrays.copyOf(buffer, Math.min(maxBytes, Math.max(buffer.length * 2, size + n))); + } + System.arraycopy(chunk, 0, buffer, size, n); + size += n; + } + return size == buffer.length ? buffer : Arrays.copyOf(buffer, size); + } + /** * Looks up a word's flag sets. * @@ -269,13 +310,13 @@ List lookup(String word) { } /** - * The suffix rules whose affix material ends in the given character, which are the + * The suffix rules whose affix material ends in the given code point, which are the * only material-bearing rules that can be undone from a word ending in it. * - * @param last The word's last character. + * @param last The word's last code point. * @return The bucket, possibly empty. Never {@code null}. */ - List suffixesEndingWith(char last) { + List suffixesEndingWith(int last) { return suffixesByLast.getOrDefault(last, NO_AFFIXES); } @@ -285,13 +326,13 @@ List suffixesWithoutMaterial() { } /** - * The prefix rules whose affix material starts with the given character, which are + * The prefix rules whose affix material starts with the given code point, which are * the only material-bearing rules that can be undone from a word starting with it. * - * @param first The word's first character. + * @param first The word's first code point. * @return The bucket, possibly empty. Never {@code null}. */ - List prefixesStartingWith(char first) { + List prefixesStartingWith(int first) { return prefixesByFirst.getOrDefault(first, NO_AFFIXES); } diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 996b2f5c08..4687283518 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -42,11 +42,13 @@ * parts ({@code ONLYINCOMPOUND}), or forbidden words ({@code FORBIDDENWORD}) never * count as standalone analyses, matching how hunspell reads those flags.

* - *

Compound part search is capped at 2048 part-licensing attempts per input word; - * beyond that budget further compound analyses are skipped. The {@link Stemmer} - * interface leaves thread safety to the implementation. This implementation reads only - * the immutable dictionary state, so a single instance is safe to share between - * threads.

+ *

Compound part search is capped at {@value #PART_CHECK_BUDGET} part-licensing + * attempts per input word; beyond that budget further compound analyses are skipped. + * The {@link Stemmer} interface leaves thread safety to the implementation. This + * implementation reads only the immutable dictionary state, so a single instance is + * safe to share between threads.

+ * + * @since 3.0.0 */ @ThreadSafe public final class HunspellStemmer implements Stemmer { @@ -146,13 +148,14 @@ private void analyze(String word, Set analyses) { if (own != null && dictionary.validStandalone(own)) { analyses.add(word); } - for (final Affix suffix : dictionary.suffixesEndingWith(word.charAt(word.length() - 1))) { + for (final Affix suffix : dictionary.suffixesEndingWith( + word.codePointBefore(word.length()))) { undoSuffix(word, suffix, analyses); } for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { undoSuffix(word, suffix, analyses); } - for (final Affix prefix : dictionary.prefixesStartingWith(word.charAt(0))) { + for (final Affix prefix : dictionary.prefixesStartingWith(word.codePointAt(0))) { undoPrefix(word, prefix, analyses); } for (final Affix prefix : dictionary.prefixesWithoutMaterial()) { @@ -348,13 +351,14 @@ private void collectPartStems(String part, CompoundPosition position, if (own != null && dictionary.mayStand(own, position)) { stems.add(part); } - for (final Affix suffix : dictionary.suffixesEndingWith(part.charAt(part.length() - 1))) { + for (final Affix suffix : dictionary.suffixesEndingWith( + part.codePointBefore(part.length()))) { collectAffixedPartStem(part, suffix, true, position, last, stems); } for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { collectAffixedPartStem(part, suffix, true, position, last, stems); } - for (final Affix prefix : dictionary.prefixesStartingWith(part.charAt(0))) { + for (final Affix prefix : dictionary.prefixesStartingWith(part.codePointAt(0))) { collectAffixedPartStem(part, prefix, false, position, first, stems); } for (final Affix prefix : dictionary.prefixesWithoutMaterial()) { @@ -437,7 +441,8 @@ private void undoSuffix(String word, Affix suffix, Set analyses) { analyses.add(stem); } } - for (final Affix inner : dictionary.suffixesEndingWith(stem.charAt(stem.length() - 1))) { + for (final Affix inner : dictionary.suffixesEndingWith( + stem.codePointBefore(stem.length()))) { undoInnerSuffix(stem, suffix, inner, analyses); } for (final Affix inner : dictionary.suffixesWithoutMaterial()) { @@ -499,7 +504,8 @@ private void undoPrefix(String word, Affix prefix, Set analyses) { if (!prefix.crossProduct()) { return; } - for (final Affix suffix : dictionary.suffixesEndingWith(stem.charAt(stem.length() - 1))) { + for (final Affix suffix : dictionary.suffixesEndingWith( + stem.codePointBefore(stem.length()))) { undoCrossProductSuffix(stem, prefix, suffix, analyses); } for (final Affix suffix : dictionary.suffixesWithoutMaterial()) { diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java index a5e995752f..b8b7c4e62e 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmerFactory.java @@ -26,6 +26,8 @@ * {@link HunspellDictionary} and hands out {@link HunspellStemmer} instances over it. * *

The factory is immutable and safe to share across threads.

+ * + * @since 3.0.0 */ @ThreadSafe public class HunspellStemmerFactory implements StemmerFactory { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 371e6ff576..8a752eb0ac 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.Assertions; @@ -1237,6 +1238,122 @@ void testResultAlteringUnsupportedDirectiveFailsLoud(String name, String line) { e.getMessage()); } + /** + * Verifies that {@link HunspellDictionary#load(InputStream, InputStream)} rejects an + * affix stream larger than {@link HunspellDictionary#MAX_STREAM_BYTES}. + */ + @Test + void testLoadRejectsOversizedAffixStream() { + final IOException e = Assertions.assertThrows(IOException.class, + () -> HunspellDictionary.load( + filledStream(HunspellDictionary.MAX_STREAM_BYTES + 1), + new ByteArrayInputStream("1\nlock\n".getBytes(StandardCharsets.UTF_8)))); + Assertions.assertEquals( + "affix stream size exceeds safe limit of " + HunspellDictionary.MAX_STREAM_BYTES, + e.getMessage()); + } + + /** + * Verifies that {@link HunspellDictionary#load(InputStream, InputStream)} rejects a + * dictionary stream larger than {@link HunspellDictionary#MAX_STREAM_BYTES}. + */ + @Test + void testLoadRejectsOversizedDictionaryStream() { + final byte[] affix = "SET UTF-8\n".getBytes(StandardCharsets.UTF_8); + final IOException e = Assertions.assertThrows(IOException.class, + () -> HunspellDictionary.load( + new ByteArrayInputStream(affix), + filledStream(HunspellDictionary.MAX_STREAM_BYTES + 1))); + Assertions.assertEquals( + "dictionary stream size exceeds safe limit of " + + HunspellDictionary.MAX_STREAM_BYTES, + e.getMessage()); + } + + /** + * Pins the inclusive stream-byte ceiling: a stream of exactly {@code limit} bytes + * succeeds, and {@code limit + 1} fails. Uses a small limit so the test does not + * allocate the production ceiling. + * + * @throws IOException Thrown if reading the in-bound stream fails. + */ + @Test + void testBoundedReadCeilingIsInclusive() throws IOException { + final int limit = 64; + final byte[] bytes = HunspellDictionary.readBounded(filledStream(limit), limit, + "affix stream"); + Assertions.assertEquals(limit, bytes.length); + final IOException e = Assertions.assertThrows(IOException.class, + () -> HunspellDictionary.readBounded(filledStream(limit + 1), limit, + "affix stream")); + Assertions.assertEquals( + "affix stream size exceeds safe limit of " + limit, e.getMessage()); + } + + /** + * Pins affix conditions and boundary bucketing to code points, matching FLAG UTF-8: + * a condition of two dots needs two code points, so a stem that is one supplementary + * character must not match, while a one-dot condition and a supplementary affix + * character still analyze. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testAffixConditionAndBoundaryUseCodePoints() throws IOException { + final HunspellStemmer twoDots = new HunspellStemmer(load( + "SFX X Y 1\nSFX X 0 s ..\n", + "1\n\uD83D\uDE00/X\n")); + Assertions.assertEquals("\uD83D\uDE00s", twoDots.stem("\uD83D\uDE00s").toString()); + + final HunspellStemmer oneDot = new HunspellStemmer(load( + "SFX X Y 1\nSFX X 0 s .\n", + "1\n\uD83D\uDE00/X\n")); + Assertions.assertEquals("\uD83D\uDE00", oneDot.stem("\uD83D\uDE00s").toString()); + + final HunspellStemmer emojiSuffix = new HunspellStemmer(load( + "SFX X Y 1\nSFX X 0 \uD83D\uDE00 .\n", + "1\nwalk/X\n")); + Assertions.assertEquals("walk", emojiSuffix.stem("walk\uD83D\uDE00").toString()); + + final HunspellStemmer classCondition = new HunspellStemmer(load( + "SFX X Y 1\nSFX X 0 s [\uD83D\uDE00]\n", + "1\nwalk\uD83D\uDE00/X\n")); + Assertions.assertEquals("walk\uD83D\uDE00", + classCondition.stem("walk\uD83D\uDE00s").toString()); + } + + /** + * Returns a stream of {@code size} zero bytes. + * + * @param size The number of bytes the stream yields. + * @return The stream. Never {@code null}. + */ + private static InputStream filledStream(int size) { + return new InputStream() { + private int remaining = size; + + @Override + public int read() { + if (remaining <= 0) { + return -1; + } + remaining--; + return 0; + } + + @Override + public int read(byte[] buffer, int offset, int length) { + if (remaining <= 0) { + return -1; + } + final int n = Math.min(length, remaining); + Arrays.fill(buffer, offset, offset + n, (byte) 0); + remaining -= n; + return n; + } + }; + } + /** * Verifies that a cosmetic unsupported directive such as {@code REP} is skipped so * the dictionary still loads. diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 96dd33ecdd..385db47422 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -97,6 +97,8 @@ stemmer.stem("table"); // "table" (unknown vocabulary is unchanged)]]> Directives that would change stems when ignored (ICONV, OCONV, COMPLEXPREFIXES) fail at load time; cosmetic tables such as REP are skipped. + Each affix or dictionary stream is rejected when it exceeds + HunspellDictionary.MAX_STREAM_BYTES (64 MiB). From 0b27d5ec1807622d5c94a3b5c882236956e09e69 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 10:53:14 -0400 Subject: [PATCH 17/24] OPENNLP-1893: Verify dictionary downloads by SHA-512 and add an opt-in URL catalog --- dev/README-hunspell-dictionaries.md | 23 ++- dev/download-hunspell-dictionary.sh | 65 ------- .../hunspell/HunspellDictionaryDownload.java | 89 +++++++++ .../opennlp/tools/util/DictionaryCatalog.java | 169 +++++++++++++++++ .../java/opennlp/tools/util/DownloadUtil.java | 170 +++++++++++++++++- .../tools/util/dictionary-catalog.properties | 57 ++++++ .../HunspellDictionaryDownloadTest.java | 60 +++++++ .../tools/util/DictionaryCatalogTest.java | 95 ++++++++++ .../opennlp/tools/util/DigestTestUtil.java | 45 +++++ .../tools/util/DownloadUtilFileTest.java | 100 +++++++++++ opennlp-docs/src/docbkx/stemmer.xml | 3 + 11 files changed, 805 insertions(+), 71 deletions(-) delete mode 100755 dev/download-hunspell-dictionary.sh create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownload.java create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java create mode 100644 opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java diff --git a/dev/README-hunspell-dictionaries.md b/dev/README-hunspell-dictionaries.md index 76175be339..b1a1f2f7de 100644 --- a/dev/README-hunspell-dictionaries.md +++ b/dev/README-hunspell-dictionaries.md @@ -23,13 +23,26 @@ The Hunspell stemmer (`opennlp.tools.stemmer.hunspell`) implements the documente The LibreOffice project maintains a large collection of Hunspell dictionaries, one directory per language, at `github.com/LibreOffice/dictionaries`. Licenses differ per dictionary, which is why nothing is bundled: for example, the `en_US` dictionary derives from SCOWL and states its terms in `README_en_US.txt` in the same directory. Many other sources work too; the engine only cares that the pair follows the Hunspell format. -The helper next to this file fetches a pair together with its readme files: +Pinned URLs and SHA-512 digests for the cataloged `en_US` pair live in +`opennlp/tools/util/dictionary-catalog.properties` (LibreOffice commit `208a9fd8`). +## Option A: opt-in catalog download + +Catalog URLs stay inactive until you set `-Dopennlp.download.remote=true`. That flag +is the explicit user action that enables the built-in URLs. + +```java +import java.nio.file.Path; +import opennlp.tools.stemmer.hunspell.HunspellDictionaryDownload; + +// JVM flag: -Dopennlp.download.remote=true +HunspellDictionaryDownload.downloadFromCatalog("en_US", Path.of("/tmp/hunspell-en_US")); ``` -./download-hunspell-dictionary.sh en en_US /tmp/hunspell-en_US -``` -## Loading and stemming +## Option B: your own files + +Fetch `.aff` / `.dic` (and the license readme) with any tool, or with +`DownloadUtil.download(uri, path, sha512)`, then load them: ```java import java.nio.file.Path; @@ -61,4 +74,4 @@ The in-tree tests run against project-authored fixtures only. An opt-in test cla ## What the engine supports -Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, the `AF` flag alias table, the `SET` encoding declaration, compound decomposition under `COMPOUNDFLAG`, the positional `COMPOUNDBEGIN`/`COMPOUNDMIDDLE`/`COMPOUNDEND` flags, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, and the `CHECKCOMPOUNDDUP`/`CHECKCOMPOUNDCASE`/`CHECKCOMPOUNDTRIPLE` declarations (compound parts stand on their entries alone or on an entry plus one affix, the zero and dash suffixes dictionaries position linking forms with included), the blocking flags `NEEDAFFIX` (alias `PSEUDOROOT`), `ONLYINCOMPOUND`, and `FORBIDDENWORD`, which keep virtual stems, compound-only parts, and forbidden words out of the reported analyses, and `CIRCUMFIX`, which binds marked prefix and suffix halves to one another as in the German `ge...t` participle. Directives that would change stems when ignored (`ICONV`, `OCONV`, `COMPLEXPREFIXES`) fail at load time. Cosmetic tables such as `REP`, `MAP`, and `KEY` are skipped, so analyses that would need them are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, the `AF` flag alias table, the `SET` encoding declaration, compound decomposition under `COMPOUNDFLAG`, the positional `COMPOUNDBEGIN`/`COMPOUNDMIDDLE`/`COMPOUNDEND` flags, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, and the `CHECKCOMPOUNDDUP`/`CHECKCOMPOUNDCASE`/`CHECKCOMPOUNDTRIPLE` declarations (compound parts stand on their entries alone or on an entry plus one affix, the zero and dash suffixes dictionaries position linking forms with included), the blocking flags `NEEDAFFIX` (alias `PSEUDOROOT`), `ONLYINCOMPOUND`, and `FORBIDDENWORD`, which keep virtual stems, compound-only parts, and forbidden words out of the reported analyses, and `CIRCUMFIX`, which binds marked prefix and suffix halves to one another as in the German `ge...t` participle. Directives that would change stems when ignored (`ICONV`, `OCONV`, `COMPLEXPREFIXES`) fail at load time. Cosmetic tables such as `REP`, `MAP`, and `KEY` are skipped, so analyses that would need them are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. Each affix or dictionary stream is rejected when it exceeds `HunspellDictionary.MAX_STREAM_BYTES` (64 MiB). diff --git a/dev/download-hunspell-dictionary.sh b/dev/download-hunspell-dictionary.sh deleted file mode 100755 index afaff59e60..0000000000 --- a/dev/download-hunspell-dictionary.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Fetches one Hunspell dictionary pair (.aff and .dic) plus its license/readme files -# from the LibreOffice dictionaries collection. Each dictionary carries its own -# license, stated in the readme files this script downloads alongside it. Apache -# OpenNLP bundles no dictionary data. See README-hunspell-dictionaries.md in this -# directory for the Java steps that follow. - -set -euo pipefail - -usage() { - echo "usage: $0 " >&2 - echo "" >&2 - echo " collection-dir the language directory inside the LibreOffice dictionaries" >&2 - echo " repository, for example: en" >&2 - echo " dictionary-name the dictionary base name inside it, for example: en_US" >&2 - echo " target-dir where the .aff, .dic, and readme files are placed" >&2 - echo "" >&2 - echo "example: $0 en en_US /tmp/hunspell-en_US" >&2 - exit 2 -} - -[ $# -ne 3 ] && usage -collection="$1" -name="$2" -target="$3" - -base="https://raw.githubusercontent.com/LibreOffice/dictionaries/master/${collection}" -mkdir -p "${target}" - -# The .aff and .dic pair is mandatory; a missing file fails the script. -for ext in aff dic; do - echo "downloading ${name}.${ext}" - curl --fail --location --silent --show-error --retry 3 \ - --output "${target}/${name}.${ext}" "${base}/${name}.${ext}" -done - -# The readme carries the dictionary's license; keep it next to the data. Different -# collections name it differently, so try the common patterns and keep what exists. -for readme in "README_${name}.txt" "README_${collection}.txt" "README.txt" "license.txt"; do - if curl --fail --location --silent --retry 3 \ - --output "${target}/${readme}" "${base}/${readme}" 2>/dev/null; then - echo "downloaded ${readme} (contains the dictionary's license; read it)" - else - rm -f "${target}/${readme}" - fi -done - -echo "" -echo "stored ${target}/${name}.aff and ${target}/${name}.dic" -echo "next: load them from Java; see README-hunspell-dictionaries.md" diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownload.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownload.java new file mode 100644 index 0000000000..51d98988da --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownload.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.IOException; +import java.nio.file.Path; + +import opennlp.tools.util.DictionaryCatalog; + +/** + * Opt-in download of Hunspell {@code .aff}/{@code .dic} pairs (and their license + * readme) from {@link DictionaryCatalog}. Requires + * {@code -Dopennlp.download.remote=true}. OpenNLP never bundles dictionary data. + * + * @since 3.0.0 + */ +public final class HunspellDictionaryDownload { + + private HunspellDictionaryDownload() { + } + + /** + * Downloads the cataloged {@code .aff}, {@code .dic}, and readme files for + * {@code dictionaryId} into {@code targetDirectory}. + * + * @param dictionaryId The catalog dictionary name, for example {@code en_US}. + * Must not be {@code null}. + * @param targetDirectory The directory to write into; created when absent. Must not + * be {@code null}. + * @throws IOException Thrown if remote downloads are disabled, a catalog entry is + * missing, or verification fails. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public static void downloadFromCatalog(String dictionaryId, Path targetDirectory) + throws IOException { + if (dictionaryId == null) { + throw new IllegalArgumentException("dictionaryId must not be null"); + } + if (targetDirectory == null) { + throw new IllegalArgumentException("targetDirectory must not be null"); + } + final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); + final String prefix = "hunspell." + dictionaryId + "."; + download(catalog, prefix + "aff", targetDirectory); + download(catalog, prefix + "dic", targetDirectory); + final String readmeId = prefix + "readme"; + if (catalog.ids().contains(readmeId)) { + download(catalog, readmeId, targetDirectory); + } + } + + /** + * Downloads one catalog entry into {@code targetDirectory}, named by the entry's + * preferred file name or, when absent, by the last segment of its URI path. + * + * @param catalog The catalog holding {@code id}. + * @param id The catalog entry id. + * @param targetDirectory The directory to write into. + * @throws IOException Thrown if remote downloads are disabled, the entry is missing, + * or the download fails verification. + */ + private static void download(DictionaryCatalog catalog, String id, Path targetDirectory) + throws IOException { + final DictionaryCatalog.Entry entry = catalog.get(id); + final String filename; + if (entry.filename() != null) { + filename = entry.filename(); + } else { + final String path = entry.uri().getPath(); + filename = path.substring(path.lastIndexOf('/') + 1); + } + catalog.download(id, targetDirectory.resolve(filename)); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java new file mode 100644 index 0000000000..5e32953346 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DictionaryCatalog.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Properties; +import java.util.Set; + +/** + * Opt-in catalog of remote dictionary archives and companion files. The catalog + * ships URLs and SHA-512 digests only; it never bundles the data itself. Fetching + * an entry requires {@link DownloadUtil#REMOTE_DOWNLOAD_PROPERTY} to be + * {@code true}, so enabling a built-in URL is an explicit user action. + * + * @since 3.0.0 + */ +public final class DictionaryCatalog { + + private static final String DEFAULT_RESOURCE = + "opennlp/tools/util/dictionary-catalog.properties"; + + private final Properties properties; + + private DictionaryCatalog(Properties properties) { + this.properties = properties; + } + + /** + * Loads the catalog shipped on the classpath. + * + * @return The catalog. Never {@code null}. + * @throws IOException Thrown if the resource is missing or cannot be read. + */ + public static DictionaryCatalog loadDefault() throws IOException { + try (InputStream in = DictionaryCatalog.class.getClassLoader() + .getResourceAsStream(DEFAULT_RESOURCE)) { + if (in == null) { + throw new IOException("missing classpath resource " + DEFAULT_RESOURCE); + } + return load(in); + } + } + + /** + * Loads a catalog from a properties stream. + * + * @param in The properties content. Must not be {@code null}. + * @return The catalog. Never {@code null}. + * @throws IOException Thrown if reading fails. + * @throws IllegalArgumentException Thrown if {@code in} is {@code null}. + */ + public static DictionaryCatalog load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("in must not be null"); + } + final Properties properties = new Properties(); + properties.load(in); + return new DictionaryCatalog(properties); + } + + /** + * {@return the catalog entry ids, in encounter order} + */ + public Set ids() { + final Set ids = new LinkedHashSet<>(); + for (final String key : properties.stringPropertyNames()) { + if (key.endsWith(".url")) { + ids.add(key.substring(0, key.length() - ".url".length())); + } + } + return Collections.unmodifiableSet(ids); + } + + /** + * Looks up one catalog entry. + * + * @param id The entry id, for example {@code mecab.ipadic}. + * @return The entry. Never {@code null}. + * @throws IOException Thrown if the entry is incomplete or the URI is malformed. + * @throws IllegalArgumentException Thrown if {@code id} is {@code null}. + */ + public Entry get(String id) throws IOException { + if (id == null) { + throw new IllegalArgumentException("id must not be null"); + } + final String url = properties.getProperty(id + ".url"); + final String sha512 = properties.getProperty(id + ".sha512"); + if (url == null || sha512 == null) { + throw new IOException("unknown or incomplete dictionary catalog entry: " + id); + } + final String filename = properties.getProperty(id + ".filename"); + try { + return new Entry(id, new URI(url), sha512.trim(), filename); + } catch (URISyntaxException e) { + throw new IOException("malformed catalog URI for " + id, e); + } + } + + /** + * Downloads a catalog entry into {@code target} after checking that remote catalog + * downloads are enabled. + * + * @param id The entry id. Must not be {@code null}. + * @param target The local file to create. Must not be {@code null}. + * @throws IOException Thrown if the property is not enabled, the entry is missing, + * or the download fails verification. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}. + */ + public void download(String id, Path target) throws IOException { + if (target == null) { + throw new IllegalArgumentException("target must not be null"); + } + if (!DownloadUtil.isRemoteDownloadEnabled()) { + throw new IOException("remote dictionary catalog downloads are disabled; set -D" + + DownloadUtil.REMOTE_DOWNLOAD_PROPERTY + "=true to enable"); + } + final Entry entry = get(id); + DownloadUtil.download(entry.uri(), target, entry.sha512()); + } + + /** + * One pinned remote file: a stable URL and the SHA-512 of its bytes. + * + * @param id The catalog id. + * @param uri The absolute download URI. + * @param sha512 The expected SHA-512 hex digest. + * @param filename An optional preferred local file name; may be {@code null}. + */ + public record Entry(String id, URI uri, String sha512, String filename) { + /** + * @param id The catalog id. Must not be {@code null}. + * @param uri The absolute download URI. Must not be {@code null}. + * @param sha512 The expected SHA-512 hex digest. Must not be {@code null}. + * @param filename An optional preferred local file name; may be {@code null}. + */ + public Entry { + if (id == null) { + throw new IllegalArgumentException("id must not be null"); + } + if (uri == null) { + throw new IllegalArgumentException("uri must not be null"); + } + if (sha512 == null) { + throw new IllegalArgumentException("sha512 must not be null"); + } + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java index 7554c064b3..44517e2389 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java @@ -21,11 +21,15 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLConnection; import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -38,6 +42,7 @@ import java.util.Formatter; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.regex.Matcher; @@ -51,7 +56,9 @@ import opennlp.tools.util.model.BaseModel; /** - * This class facilitates the downloading of pretrained OpenNLP models. + * Downloads remote resources into a local path: pretrained OpenNLP models, and any + * other file fetched through {@link #download(URI, Path, String)} with an expected + * SHA-512 digest. */ public class DownloadUtil { @@ -63,6 +70,22 @@ public class DownloadUtil { System.getProperty("OPENNLP_DOWNLOAD_MODEL_PATH", "models/ud-models-1.3/"); private static final String OPENNLP_DOWNLOAD_HOME = "OPENNLP_DOWNLOAD_HOME"; + /** + * System property that must be {@code true} before a + * {@link DictionaryCatalog} entry may be fetched. Explicit + * {@link #download(URI, Path, String)} calls do not require it: the caller already + * supplied the URI and digest. + */ + public static final String REMOTE_DOWNLOAD_PROPERTY = "opennlp.download.remote"; + + /** Inclusive ceiling on bytes buffered for one {@link #download(URI, Path, String)}. */ + public static final long MAX_DOWNLOAD_BYTES = 512L * 1024 * 1024; + + private static final int CONNECT_TIMEOUT_MS = 30_000; + private static final int READ_TIMEOUT_MS = 300_000; + private static final int SHA512_HEX_LENGTH = 128; + private static final String DOWNLOAD_SUFFIX = ".download"; + private static Map> availableModels; /** @@ -172,6 +195,151 @@ public static T downloadModel(URL url, Class type) thro } } + /** + * Downloads {@code source} into {@code target} and requires the SHA-512 digest of the + * stored bytes to equal {@code expectedSha512}. The download is written to a sibling + * temporary file and moved into place only after the digest matches. The transfer is + * capped at {@link #MAX_DOWNLOAD_BYTES}; remote {@code http} and {@code https} URIs + * additionally use connect and read timeouts. + * + * @param source The absolute URI to fetch. Must not be {@code null}. + * @param target The local file to create or replace. Must not be {@code null}. + * @param expectedSha512 The expected SHA-512 digest as 128 lowercase or uppercase hex + * digits. Must not be {@code null}. + * @throws IOException Thrown if fetching fails, the size ceiling is exceeded, or the + * digest does not match. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, {@code source} + * is not absolute, or {@code expectedSha512} is not 128 hex digits. + */ + public static void download(URI source, Path target, String expectedSha512) + throws IOException { + download(source, target, expectedSha512, MAX_DOWNLOAD_BYTES); + } + + /** + * Downloads {@code source} into {@code target} under a caller-supplied byte ceiling. + * + * @param source The absolute URI to fetch. Must not be {@code null}. + * @param target The local file to create or replace. Must not be {@code null}. + * @param expectedSha512 The expected SHA-512 digest as 128 hex digits. Must not be + * {@code null}. + * @param maxBytes The inclusive ceiling on bytes read from {@code source}. + * @throws IOException Thrown if fetching fails, {@code maxBytes} is exceeded, or the + * digest does not match. + * @throws IllegalArgumentException Thrown if a parameter is invalid, see + * {@link #download(URI, Path, String)}. + */ + static void download(URI source, Path target, String expectedSha512, long maxBytes) + throws IOException { + if (source == null) { + throw new IllegalArgumentException("source must not be null"); + } + if (target == null) { + throw new IllegalArgumentException("target must not be null"); + } + if (expectedSha512 == null) { + throw new IllegalArgumentException("expectedSha512 must not be null"); + } + if (!source.isAbsolute()) { + throw new IllegalArgumentException("source must be an absolute URI"); + } + final String normalized = normalizeSha512(expectedSha512); + final Path parent = target.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + final Path partial = target.resolveSibling(target.getFileName() + DOWNLOAD_SUFFIX); + Files.deleteIfExists(partial); + try { + long size = 0L; + final MessageDigest digest = sha512Digest(); + final URLConnection connection = open(source); + try (InputStream in = connection.getInputStream(); + DigestInputStream digester = new DigestInputStream(in, digest); + OutputStream out = Files.newOutputStream(partial)) { + final byte[] buffer = new byte[8192]; + int n; + while ((n = digester.read(buffer)) >= 0) { + size += n; + if (size > maxBytes) { + throw new IOException("download size exceeds safe limit of " + maxBytes); + } + out.write(buffer, 0, n); + } + } finally { + if (connection instanceof HttpURLConnection http) { + http.disconnect(); + } + } + final String actual = byteArrayToHexString(digest.digest()); + if (!actual.equals(normalized)) { + throw new IOException("SHA512 checksum validation failed for " + target.getFileName() + + ". Expected: " + normalized + ", but got: " + actual); + } + try { + Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(partial, target, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + Files.deleteIfExists(partial); + throw e; + } + } + + /** + * {@return {@code true} when {@link #REMOTE_DOWNLOAD_PROPERTY} is the string + * {@code true}, ignoring case} + */ + public static boolean isRemoteDownloadEnabled() { + return Boolean.parseBoolean(System.getProperty(REMOTE_DOWNLOAD_PROPERTY)); + } + + /** + * Opens a connection to {@code source} with connect and read timeouts applied. + * + * @param source The absolute URI to connect to. + * @return The configured, not yet connected, connection. + * @throws IOException Thrown if no connection can be created for {@code source}. + */ + private static URLConnection open(URI source) throws IOException { + final URLConnection connection = source.toURL().openConnection(); + connection.setConnectTimeout(CONNECT_TIMEOUT_MS); + connection.setReadTimeout(READ_TIMEOUT_MS); + return connection; + } + + /** + * Trims and lowercases a SHA-512 hex digest. + * + * @param expectedSha512 The digest to normalize. + * @return The digest as 128 lowercase hex digits. + * @throws IllegalArgumentException Thrown if the digest is not 128 hex digits. + */ + private static String normalizeSha512(String expectedSha512) { + final String hex = expectedSha512.trim().toLowerCase(Locale.ROOT); + if (hex.length() != SHA512_HEX_LENGTH || !hex.chars().allMatch( + c -> c >= '0' && c <= '9' || c >= 'a' && c <= 'f')) { + throw new IllegalArgumentException( + "expectedSha512 must be 128 hexadecimal digits"); + } + return hex; + } + + /** + * {@return a fresh SHA-512 {@link MessageDigest}} + * + * @throws IOException Thrown if the JVM does not provide the algorithm. + */ + private static MessageDigest sha512Digest() throws IOException { + try { + return MessageDigest.getInstance("SHA-512"); + } catch (NoSuchAlgorithmException e) { + throw new IOException("SHA-512 algorithm not found", e); + } + } + public static Map> getAvailableModels() { if (availableModels == null) { try { diff --git a/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties new file mode 100644 index 0000000000..b016826d89 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/dictionary-catalog.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pinned remote dictionary files. OpenNLP ships URLs and SHA-512 digests only; +# the data itself is never bundled. Fetching requires -Dopennlp.download.remote=true. + +# MeCab IPADIC 2.7.0 (EUC-JP). Upstream: MeCab project on SourceForge. +mecab.ipadic.url=https://downloads.sourceforge.net/project/mecab/\ +mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz +mecab.ipadic.sha512=35ea662cb62f1967849f7ed5781bd6dafef0fe20d63e88d9\ +a0057666e57ed23d5a0e6fb8d0701a0cc4da43a1050c1b0246\ +3bb862decc71c36b7fc2acdc158d86 +mecab.ipadic.filename=mecab-ipadic-2.7.0-20070801.tar.gz + +# mecab-ko-dic 2.1.1 (UTF-8). Upstream: eunjeon/mecab-ko-dic on Bitbucket. +mecab.ko-dic.url=https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/\ +mecab-ko-dic-2.1.1-20180720.tar.gz +mecab.ko-dic.sha512=986f8f9c66c53accd296756bf632c979d2d44b695ada33f3\ +6c662f210dba34cd95d67b61dd8c84a1f7d59f80ee6bc22eb1\ +e9afb5dc6a7f9b6b75b4fbf2f8164f +mecab.ko-dic.filename=mecab-ko-dic-2.1.1-20180720.tar.gz + +# LibreOffice en_US Hunspell pair, pinned to dictionaries commit 208a9fd8. +hunspell.en_US.aff.url=https://raw.githubusercontent.com/LibreOffice/\ +dictionaries/208a9fd80b2a182fe20f224cd615119c6323ae2e/en/en_US.aff +hunspell.en_US.aff.sha512=2b4448dfdff03caf300914415f4642f8d2ba5b650c5f024a\ +12355b420a279ffc12146649fce092ba591504476634a3d6\ +fd4c079335a27085b396fa76bfd28b74 +hunspell.en_US.aff.filename=en_US.aff + +hunspell.en_US.dic.url=https://raw.githubusercontent.com/LibreOffice/\ +dictionaries/208a9fd80b2a182fe20f224cd615119c6323ae2e/en/en_US.dic +hunspell.en_US.dic.sha512=4be737249a8a436d20a02be575dcf6cf2f06f5f2abb840ea\ +5ec0ef0ac73a71fa0e4669e527c703d5c6b50ef61713a674\ +1b55bc136d559a54bcdeebcd62027988 +hunspell.en_US.dic.filename=en_US.dic + +hunspell.en_US.readme.url=https://raw.githubusercontent.com/LibreOffice/\ +dictionaries/208a9fd80b2a182fe20f224cd615119c6323ae2e/en/README_en_US.txt +hunspell.en_US.readme.sha512=aa23ebc8adc0649b540264c7bf98cef5b6e383fec0e4a1a7\ +dd49d1c887cfeefd8edf6a568afc8a651521a3864c5b1ab5\ +0748ad16230d386809080b5b09135082 +hunspell.en_US.readme.filename=README_en_US.txt diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java new file mode 100644 index 0000000000..ae064ce69a --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.stemmer.hunspell; + +import java.io.IOException; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.DictionaryCatalog; +import opennlp.tools.util.DownloadUtil; + +/** + * Pins the Hunspell catalog download gate; network fetches are not exercised here. + */ +public class HunspellDictionaryDownloadTest { + + @Test + void testDownloadRequiresRemoteProperty(@TempDir Path target) { + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + try { + final IOException e = Assertions.assertThrows(IOException.class, + () -> HunspellDictionaryDownload.downloadFromCatalog("en_US", target)); + Assertions.assertTrue(e.getMessage().contains(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY)); + } finally { + if (previous == null) { + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + } else { + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, previous); + } + } + } + + @Test + void testCatalogContainsEnUsPair() throws IOException { + final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); + Assertions.assertTrue(catalog.ids().contains("hunspell.en_US.aff")); + Assertions.assertTrue(catalog.ids().contains("hunspell.en_US.dic")); + Assertions.assertTrue(catalog.ids().contains("hunspell.en_US.readme")); + Assertions.assertEquals(128, catalog.get("hunspell.en_US.aff").sha512().length()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java new file mode 100644 index 0000000000..fd05fd6ed5 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests the opt-in dictionary catalog against an in-memory properties file and a + * local file URI so no network access is required. + */ +public class DictionaryCatalogTest { + + @Test + void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception { + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + final DictionaryCatalog loaded = demoCatalog(dir, payload); + + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + try { + final IOException e = Assertions.assertThrows(IOException.class, + () -> loaded.download("demo", dir.resolve("out.bin"))); + Assertions.assertTrue(e.getMessage().contains(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY)); + } finally { + restore(previous); + } + } + + @Test + void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception { + final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + final DictionaryCatalog loaded = demoCatalog(dir, payload); + final Path target = dir.resolve("out.bin"); + + final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, "true"); + try { + loaded.download("demo", target); + Assertions.assertArrayEquals(payload, Files.readAllBytes(target)); + } finally { + restore(previous); + } + } + + @Test + void testDefaultCatalogContainsMecabAndHunspellEntries() throws IOException { + final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); + Assertions.assertTrue(catalog.ids().contains("mecab.ipadic")); + Assertions.assertTrue(catalog.ids().contains("mecab.ko-dic")); + Assertions.assertTrue(catalog.ids().contains("hunspell.en_US.aff")); + Assertions.assertEquals(128, catalog.get("mecab.ipadic").sha512().length()); + Assertions.assertEquals(128, catalog.get("hunspell.en_US.dic").sha512().length()); + } + + private static DictionaryCatalog demoCatalog(Path dir, byte[] payload) + throws IOException { + final Path source = dir.resolve("dict.bin"); + Files.write(source, payload); + final String catalog = "demo.url=" + source.toUri() + "\n" + + "demo.sha512=" + DigestTestUtil.sha512(payload) + "\n"; + return DictionaryCatalog.load( + new ByteArrayInputStream(catalog.getBytes(StandardCharsets.UTF_8))); + } + + private static void restore(String previous) { + if (previous == null) { + System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); + } else { + System.setProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY, previous); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java new file mode 100644 index 0000000000..af85b09a8e --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DigestTestUtil.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +/** + * Computes SHA-512 digests for test fixtures. + */ +public final class DigestTestUtil { + + private DigestTestUtil() { + } + + /** + * {@return the SHA-512 digest of {@code bytes} as 128 lowercase hex digits} + * + * @param bytes The content to digest. Must not be {@code null}. + */ + public static String sha512(byte[] bytes) { + try { + return HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-512").digest(bytes)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java new file mode 100644 index 0000000000..32510fcb35 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Pins {@link DownloadUtil#download(java.net.URI, Path, String)} against local file URIs + * so digest verification and the size ceiling are covered without a network. + */ +public class DownloadUtilFileTest { + + private static final byte[] PAYLOAD = "dictionary-bytes".getBytes(StandardCharsets.UTF_8); + + @Test + void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + DownloadUtil.download(source.toUri(), target, DigestTestUtil.sha512(PAYLOAD)); + + Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); + } + + @Test + void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + final String wrong = DigestTestUtil.sha512("other".getBytes(StandardCharsets.UTF_8)); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> DownloadUtil.download(source.toUri(), target, wrong)); + Assertions.assertTrue(e.getMessage().contains("SHA512 checksum validation failed")); + Assertions.assertTrue(Files.notExists(target)); + } + + @Test + void testDownloadRequiresSha512() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> DownloadUtil.download(Path.of("x").toUri(), Path.of("y"), null)); + } + + @Test + void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> DownloadUtil.download(source.toUri(), dir.resolve("target.bin"), "abc123")); + } + + @Test + void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + final IOException e = Assertions.assertThrows(IOException.class, + () -> DownloadUtil.download(source.toUri(), target, + DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length - 1)); + Assertions.assertTrue(e.getMessage().contains("exceeds safe limit")); + Assertions.assertTrue(Files.notExists(target)); + } + + @Test + void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { + final Path source = dir.resolve("source.bin"); + Files.write(source, PAYLOAD); + final Path target = dir.resolve("target.bin"); + + DownloadUtil.download(source.toUri(), target, + DigestTestUtil.sha512(PAYLOAD), PAYLOAD.length); + + Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); + } +} diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 385db47422..87bf1b0eda 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -94,6 +94,9 @@ stemmer.stem("table"); // "table" (unknown vocabulary is unchanged)]]> published dictionary yields for a given form is decided by that dictionary. Acquisition helpers and the supported affix feature set live in dev/README-hunspell-dictionaries.md. + An opt-in catalog download + (HunspellDictionaryDownload.downloadFromCatalog) needs + -Dopennlp.download.remote=true and verifies SHA-512 digests. Directives that would change stems when ignored (ICONV, OCONV, COMPLEXPREFIXES) fail at load time; cosmetic tables such as REP are skipped. From 39afc7e59ff6011be9ba0a4aeb81aa1b82c35f07 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 16:12:07 -0400 Subject: [PATCH 18/24] OPENNLP-1893: Sync shared DownloadUtil with the startup-overridable download ceiling The 512 MiB download ceiling becomes a default that opennlp.download.max.bytes can raise at JVM startup; absent or invalid values fall back. Keeps the file identical to the copy in the MeCab PR. --- .../java/opennlp/tools/util/DownloadUtil.java | 40 ++++++++++++++++++- .../tools/util/DownloadUtilFileTest.java | 36 +++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java index 44517e2389..41593e97d4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java @@ -78,8 +78,20 @@ public class DownloadUtil { */ public static final String REMOTE_DOWNLOAD_PROPERTY = "opennlp.download.remote"; - /** Inclusive ceiling on bytes buffered for one {@link #download(URI, Path, String)}. */ - public static final long MAX_DOWNLOAD_BYTES = 512L * 1024 * 1024; + /** + * System property for overriding {@link #MAX_DOWNLOAD_BYTES}. Set at JVM startup, + * e.g. {@code -Dopennlp.download.max.bytes=2147483648} for dictionaries larger than + * the default ceiling. Falls back to the default if absent, non-numeric, or not + * positive. + */ + public static final String MAX_DOWNLOAD_BYTES_PROPERTY = "opennlp.download.max.bytes"; + + /** + * Inclusive ceiling on bytes buffered for one {@link #download(URI, Path, String)}, + * 512 MiB unless overridden via {@link #MAX_DOWNLOAD_BYTES_PROPERTY}. + */ + public static final long MAX_DOWNLOAD_BYTES = + configuredLimit(MAX_DOWNLOAD_BYTES_PROPERTY, 512L * 1024 * 1024); private static final int CONNECT_TIMEOUT_MS = 30_000; private static final int READ_TIMEOUT_MS = 300_000; @@ -296,6 +308,30 @@ public static boolean isRemoteDownloadEnabled() { return Boolean.parseBoolean(System.getProperty(REMOTE_DOWNLOAD_PROPERTY)); } + /** + * Reads a byte-budget override from a system property. Budget constants are + * initialized from it once at class load, so overrides must be set at JVM startup. + * + * @param property The system property name to read. + * @param fallback The value to use when the property is absent or invalid. + * @return The property's value when it parses as a positive {@code long}, otherwise + * {@code fallback}. + */ + public static long configuredLimit(String property, long fallback) { + final String value = System.getProperty(property, "").trim(); + if (!value.isEmpty()) { + try { + final long parsed = Long.parseLong(value); + if (parsed > 0) { + return parsed; + } + } catch (NumberFormatException ignore) { + // Fall through to the default. + } + } + return fallback; + } + /** * Opens a connection to {@code source} with connect and read timeouts applied. * diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java index 32510fcb35..113f1689cf 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -97,4 +97,40 @@ void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); } + + @Test + void testConfiguredLimitOverridesFromProperty() { + final String property = "opennlp.test.limit.override"; + System.setProperty(property, "1024"); + try { + Assertions.assertEquals(1024L, DownloadUtil.configuredLimit(property, 7L)); + } finally { + System.clearProperty(property); + } + } + + @Test + void testConfiguredLimitFallsBackWhenAbsent() { + Assertions.assertEquals(7L, + DownloadUtil.configuredLimit("opennlp.test.limit.absent", 7L)); + } + + @Test + void testConfiguredLimitRejectsInvalidValues() { + final String property = "opennlp.test.limit.invalid"; + for (final String invalid : new String[] {"", " ", "abc", "-1", "0"}) { + System.setProperty(property, invalid); + try { + Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L), + "value <" + invalid + "> must fall back"); + } finally { + System.clearProperty(property); + } + } + } + + @Test + void testDefaultBudgetsWithoutOverrides() { + Assertions.assertEquals(512L * 1024 * 1024, DownloadUtil.MAX_DOWNLOAD_BYTES); + } } From 722f3bdf433b971331ea3f12765d59da3f59ab61 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 19:11:45 -0400 Subject: [PATCH 19/24] OPENNLP-1893: Trigger CI for the DownloadUtil sync commit From 43a782513d40169f970df097068c35838432d468 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 19:01:35 -0400 Subject: [PATCH 20/24] OPENNLP-1893: Address review: unbox the boundary lookups and publish the stream ceiling The affix boundary buckets move from a boxed Integer map to a sorted int index answered by binary search, closing the per-call boxing note from the review; the strip-only rule lists are frozen at load so no internal mutable list is handed out. MAX_STREAM_BYTES becomes public, matching its citation in the load javadoc, the manual chapter, and the dev README. A parameterized test pins trimming of word-list lines edged by the no-break space and the ideographic space on both edges, and the download and catalog test classes get the javadoc coverage the hunspell tests already carry. --- .../stemmer/hunspell/HunspellDictionary.java | 68 +++++++++++++++---- .../HunspellDictionaryDownloadTest.java | 12 ++++ .../stemmer/hunspell/HunspellStemmerTest.java | 22 ++++-- .../tools/util/DictionaryCatalogTest.java | 34 ++++++++++ .../tools/util/DownloadUtilFileTest.java | 41 +++++++++++ 5 files changed, 159 insertions(+), 18 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 2c3123a952..405bcbe46b 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -75,7 +75,7 @@ public final class HunspellDictionary { * during {@link #load(InputStream, InputStream)}. Larger streams fail with * {@link IOException}. */ - static final int MAX_STREAM_BYTES = 64 * 1024 * 1024; + public static final int MAX_STREAM_BYTES = 64 * 1024 * 1024; /** * One parsed affix rule of a {@code PFX} or {@code SFX} block. @@ -129,9 +129,9 @@ enum CompoundPosition { private static final String NO_MATERIAL = "0"; private final Map> entries; - private final Map> suffixesByLast; + private final BoundaryIndex suffixesByLast; private final List suffixesWithoutMaterial; - private final Map> prefixesByFirst; + private final BoundaryIndex prefixesByFirst; private final List prefixesWithoutMaterial; private final int compoundFlag; private final int compoundBegin; @@ -175,10 +175,54 @@ private HunspellDictionary(Map> entries, AffixFile affix) { // A material-bearing rule can only be undone from a word whose boundary // character matches its affix material, so bucketing by that character // narrows each scan to one bucket plus the strip-only rules. - this.suffixesWithoutMaterial = new ArrayList<>(); - this.suffixesByLast = bucketByBoundary(affix.suffixes, true, suffixesWithoutMaterial); - this.prefixesWithoutMaterial = new ArrayList<>(); - this.prefixesByFirst = bucketByBoundary(affix.prefixes, false, prefixesWithoutMaterial); + final List suffixesWithout = new ArrayList<>(); + this.suffixesByLast = bucketByBoundary(affix.suffixes, true, suffixesWithout); + this.suffixesWithoutMaterial = List.copyOf(suffixesWithout); + final List prefixesWithout = new ArrayList<>(); + this.prefixesByFirst = bucketByBoundary(affix.prefixes, false, prefixesWithout); + this.prefixesWithoutMaterial = List.copyOf(prefixesWithout); + } + + /** + * An immutable index of affix rules keyed by the boundary code point of their affix + * material, answering each lookup by binary search so the per-word scans in + * {@link HunspellStemmer} allocate nothing. + */ + private static final class BoundaryIndex { + + /** The boundary code points, sorted ascending. */ + private final int[] boundaries; + /** The rule bucket for each boundary, aligned with {@link #boundaries}. */ + private final List> buckets; + + /** + * Initializes the index from mutable buckets, freezing each one. + * + * @param byBoundary The rule buckets keyed by boundary code point. + */ + private BoundaryIndex(Map> byBoundary) { + this.boundaries = new int[byBoundary.size()]; + int b = 0; + for (final Integer boundary : byBoundary.keySet()) { + boundaries[b++] = boundary; + } + Arrays.sort(boundaries); + this.buckets = new ArrayList<>(boundaries.length); + for (final int boundary : boundaries) { + buckets.add(List.copyOf(byBoundary.get(boundary))); + } + } + + /** + * The rules bucketed under a boundary code point. + * + * @param codePoint The boundary code point to look up. + * @return The bucket, possibly empty. Never {@code null}. + */ + List bucket(int codePoint) { + final int index = Arrays.binarySearch(boundaries, codePoint); + return index >= 0 ? buckets.get(index) : NO_AFFIXES; + } } /** @@ -189,9 +233,9 @@ private HunspellDictionary(Map> entries, AffixFile affix) { * @param suffix Whether the rules are suffix rules. * @param withoutMaterial Collects the rules with empty affix material, which no * boundary code point keys. - * @return The rules keyed by their boundary code point. Never {@code null}. + * @return The rules indexed by their boundary code point. Never {@code null}. */ - private static Map> bucketByBoundary(List rules, + private static BoundaryIndex bucketByBoundary(List rules, boolean suffix, List withoutMaterial) { final Map> byBoundary = new HashMap<>(); for (final Affix rule : rules) { @@ -205,7 +249,7 @@ private static Map> bucketByBoundary(List rules, byBoundary.computeIfAbsent(boundary, key -> new ArrayList<>()).add(rule); } } - return byBoundary; + return new BoundaryIndex(byBoundary); } /** @@ -317,7 +361,7 @@ List lookup(String word) { * @return The bucket, possibly empty. Never {@code null}. */ List suffixesEndingWith(int last) { - return suffixesByLast.getOrDefault(last, NO_AFFIXES); + return suffixesByLast.bucket(last); } /** {@return the strip-only suffix rules, applicable to any word} Never {@code null}. */ @@ -333,7 +377,7 @@ List suffixesWithoutMaterial() { * @return The bucket, possibly empty. Never {@code null}. */ List prefixesStartingWith(int first) { - return prefixesByFirst.getOrDefault(first, NO_AFFIXES); + return prefixesByFirst.bucket(first); } /** {@return the strip-only prefix rules, applicable to any word} Never {@code null}. */ diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java index ae064ce69a..15d576a5f1 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellDictionaryDownloadTest.java @@ -32,6 +32,12 @@ */ public class HunspellDictionaryDownloadTest { + /** + * Verifies that a catalog download without the remote-download property fails with + * the property name in the message, leaving the previous property value restored. + * + * @param target A scratch directory managed by the test framework. + */ @Test void testDownloadRequiresRemoteProperty(@TempDir Path target) { final String previous = System.getProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); @@ -49,6 +55,12 @@ void testDownloadRequiresRemoteProperty(@TempDir Path target) { } } + /** + * Verifies that the shipped catalog holds the {@code en_US} pair and its license + * readme, each with a full-length SHA-512 digest. + * + * @throws IOException Thrown if the shipped catalog fails to load. + */ @Test void testCatalogContainsEnUsPair() throws IOException { final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 8a752eb0ac..4e50afd018 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -30,6 +30,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import opennlp.tools.stemmer.Stemmer; @@ -68,6 +69,11 @@ public class HunspellStemmerTest { private static HunspellStemmer stemmer; + /** + * Loads the shared fixture dictionary once for the tests that stem against it. + * + * @throws IOException Thrown if the fixture fails to load. + */ @BeforeAll static void loadDictionary() throws IOException { stemmer = new HunspellStemmer(load(AFFIX, WORDS)); @@ -482,16 +488,20 @@ void testMultiWordEntriesKeepTheirSpacesAndFlags() throws IOException { /** * Verifies that the parser trims word-list entries with the same whitespace - * definition it uses to find their fields: an entry led by a no-break space is - * registered under its real word, both with and without a flag run. + * definition it uses to find their fields: an entry edged by Unicode whitespace, + * leading or trailing, is registered under its real word, both with and without a + * flag run. * + * @param space The whitespace character at the line edges: the no-break space + * U+00A0 and the ideographic space U+3000, both whitespace to + * {@code StringUtil.isWhitespace} but not to {@code String.trim()}. * @throws IOException Thrown if a fixture fails to load. */ - @Test - void testEntriesLedByNoBreakSpaceAreTrimmed() throws IOException { - // \u00A0 is the no-break space, which StringUtil.isWhitespace treats as whitespace + @ParameterizedTest + @ValueSource(strings = {"\u00A0", "\u3000"}) + void testEntriesEdgedByUnicodeWhitespaceAreTrimmed(String space) throws IOException { final HunspellDictionary dictionary = load("SFX S Y 1\nSFX S 0 s .\n", - "2\n\u00A0fish\n\u00A0cat/S\n"); + "2\n" + space + "fish" + space + "\n" + space + "cat/S" + space + "\n"); Assertions.assertNotNull(dictionary.lookup("fish")); Assertions.assertNotNull(dictionary.lookup("cat")); Assertions.assertNull(dictionary.lookup("")); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java index fd05fd6ed5..d721873951 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DictionaryCatalogTest.java @@ -33,6 +33,13 @@ */ public class DictionaryCatalogTest { + /** + * Verifies that a catalog download without the remote-download property fails with + * the property name in the message. + * + * @param dir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture catalog cannot be prepared. + */ @Test void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception { final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); @@ -49,6 +56,13 @@ void testDownloadRequiresRemoteProperty(@TempDir Path dir) throws Exception { } } + /** + * Verifies that an enabled catalog download fetches the entry and writes the + * digest-verified bytes to the target. + * + * @param dir A scratch directory managed by the test framework. + * @throws Exception Thrown if the fixture catalog cannot be prepared or fetched. + */ @Test void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception { final byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); @@ -65,6 +79,12 @@ void testDownloadWithRemotePropertyEnabled(@TempDir Path dir) throws Exception { } } + /** + * Verifies that the shipped catalog holds the MeCab and Hunspell entries, each with + * a full-length SHA-512 digest. + * + * @throws IOException Thrown if the shipped catalog fails to load. + */ @Test void testDefaultCatalogContainsMecabAndHunspellEntries() throws IOException { final DictionaryCatalog catalog = DictionaryCatalog.loadDefault(); @@ -75,6 +95,15 @@ void testDefaultCatalogContainsMecabAndHunspellEntries() throws IOException { Assertions.assertEquals(128, catalog.get("hunspell.en_US.dic").sha512().length()); } + /** + * Builds a one-entry catalog whose URL is a local file holding {@code payload}, so + * downloads need no network. + * + * @param dir The directory to write the payload file into. + * @param payload The bytes the catalog entry points at. + * @return The loaded catalog. Never {@code null}. + * @throws IOException Thrown if the payload file cannot be written. + */ private static DictionaryCatalog demoCatalog(Path dir, byte[] payload) throws IOException { final Path source = dir.resolve("dict.bin"); @@ -85,6 +114,11 @@ private static DictionaryCatalog demoCatalog(Path dir, byte[] payload) new ByteArrayInputStream(catalog.getBytes(StandardCharsets.UTF_8))); } + /** + * Restores the remote-download property to its value before the test. + * + * @param previous The saved value, or {@code null} when the property was unset. + */ private static void restore(String previous) { if (previous == null) { System.clearProperty(DownloadUtil.REMOTE_DOWNLOAD_PROPERTY); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java index 113f1689cf..463be3b3f7 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -32,8 +32,16 @@ */ public class DownloadUtilFileTest { + /** The fixture bytes the download tests serve and digest. */ private static final byte[] PAYLOAD = "dictionary-bytes".getBytes(StandardCharsets.UTF_8); + /** + * Verifies that a download whose bytes match the expected digest lands in the + * target file. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or fetched. + */ @Test void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -45,6 +53,13 @@ void testDownloadAcceptsMatchingDigest(@TempDir Path dir) throws IOException { Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); } + /** + * Verifies that a digest mismatch fails the download and leaves no target file + * behind. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ @Test void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -58,12 +73,20 @@ void testDownloadRejectsMismatchedDigest(@TempDir Path dir) throws IOException { Assertions.assertTrue(Files.notExists(target)); } + /** Verifies that a {@code null} digest is rejected with the documented exception. */ @Test void testDownloadRequiresSha512() { Assertions.assertThrows(IllegalArgumentException.class, () -> DownloadUtil.download(Path.of("x").toUri(), Path.of("y"), null)); } + /** + * Verifies that a digest shorter than 128 hex digits is rejected before anything is + * fetched. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ @Test void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -73,6 +96,13 @@ void testDownloadRejectsMalformedSha512(@TempDir Path dir) throws IOException { () -> DownloadUtil.download(source.toUri(), dir.resolve("target.bin"), "abc123")); } + /** + * Verifies that a source larger than the byte ceiling fails the download and leaves + * no target file behind. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written. + */ @Test void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -86,6 +116,13 @@ void testDownloadRejectsOversizedSource(@TempDir Path dir) throws IOException { Assertions.assertTrue(Files.notExists(target)); } + /** + * Pins the inclusive byte ceiling: a source of exactly the ceiling's size still + * downloads. + * + * @param dir A scratch directory managed by the test framework. + * @throws IOException Thrown if the fixture cannot be written or fetched. + */ @Test void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { final Path source = dir.resolve("source.bin"); @@ -98,6 +135,7 @@ void testDownloadCeilingIsInclusive(@TempDir Path dir) throws IOException { Assertions.assertArrayEquals(PAYLOAD, Files.readAllBytes(target)); } + /** Verifies that a positive property value overrides the fallback limit. */ @Test void testConfiguredLimitOverridesFromProperty() { final String property = "opennlp.test.limit.override"; @@ -109,12 +147,14 @@ void testConfiguredLimitOverridesFromProperty() { } } + /** Verifies that an unset property falls back to the given default. */ @Test void testConfiguredLimitFallsBackWhenAbsent() { Assertions.assertEquals(7L, DownloadUtil.configuredLimit("opennlp.test.limit.absent", 7L)); } + /** Verifies that blank, non-numeric, and non-positive values fall back. */ @Test void testConfiguredLimitRejectsInvalidValues() { final String property = "opennlp.test.limit.invalid"; @@ -129,6 +169,7 @@ void testConfiguredLimitRejectsInvalidValues() { } } + /** Pins the default download ceiling of 512 MiB when no override property is set. */ @Test void testDefaultBudgetsWithoutOverrides() { Assertions.assertEquals(512L * 1024 * 1024, DownloadUtil.MAX_DOWNLOAD_BYTES); From e6d39bd6f4eed98cf01b6214dc798f0ab4761855 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 19:26:47 -0400 Subject: [PATCH 21/24] OPENNLP-1893: Use a numeric character reference for the no-break space in the manual DocBook XML defines no nbsp entity, so the PDF build rejects it. --- opennlp-docs/src/docbkx/stemmer.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 87bf1b0eda..8b02a2f607 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -101,7 +101,7 @@ stemmer.stem("table"); // "table" (unknown vocabulary is unchanged)]]> (ICONV, OCONV, COMPLEXPREFIXES) fail at load time; cosmetic tables such as REP are skipped. Each affix or dictionary stream is rejected when it exceeds - HunspellDictionary.MAX_STREAM_BYTES (64 MiB). + HunspellDictionary.MAX_STREAM_BYTES (64 MiB). From 9f811674f57ba9fcfa73edae62e8a5b33328c229 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:19:40 -0400 Subject: [PATCH 22/24] OPENNLP-1893: Reconcile the shared download test files with the sibling PR Both PRs carry byte-identical copies; this folds the sibling's parameterized invalid-limit test into this side's documented fixtures so the copies match again. --- .../tools/util/DownloadUtilFileTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java index 463be3b3f7..5451cabc5f 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/DownloadUtilFileTest.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Pins {@link DownloadUtil#download(java.net.URI, Path, String)} against local file URIs @@ -155,17 +157,15 @@ void testConfiguredLimitFallsBackWhenAbsent() { } /** Verifies that blank, non-numeric, and non-positive values fall back. */ - @Test - void testConfiguredLimitRejectsInvalidValues() { + @ParameterizedTest(name = "value \"{0}\" falls back") + @ValueSource(strings = {"", " ", "abc", "-1", "0"}) + void testConfiguredLimitRejectsInvalidValues(String invalid) { final String property = "opennlp.test.limit.invalid"; - for (final String invalid : new String[] {"", " ", "abc", "-1", "0"}) { - System.setProperty(property, invalid); - try { - Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L), - "value <" + invalid + "> must fall back"); - } finally { - System.clearProperty(property); - } + System.setProperty(property, invalid); + try { + Assertions.assertEquals(7L, DownloadUtil.configuredLimit(property, 7L)); + } finally { + System.clearProperty(property); } } From fe53f4c9974d97b31a78b41d101f088411ebac06 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 08:34:09 -0400 Subject: [PATCH 23/24] OPENNLP-1893: Expose silent COMPOUNDRULE, IGNORE, KEEPCASE and ungated full-strip rules with failing tests COMPOUNDRULE, IGNORE, and KEEPCASE alter analyses when ignored, so the fail-closed loader policy requires them to fail at load time like ICONV, OCONV, and COMPLEXPREFIXES; the loader currently accepts them silently. A suffix rule whose strip string is the whole stem is applied without the FULLSTRIP declaration hunspell requires for it, inventing a stem for a surface form the dictionary does not license. --- .../stemmer/hunspell/HunspellStemmerTest.java | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java index 4e50afd018..8657b7e8c0 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/hunspell/HunspellStemmerTest.java @@ -1232,14 +1232,18 @@ void testForbiddenEntryBlocksItsDecomposition() throws IOException { /** * Verifies that result-altering unsupported affix directives fail at load time. - * Ignoring {@code ICONV}, {@code OCONV}, or {@code COMPLEXPREFIXES} would change - * stems with no signal. + * Ignoring {@code ICONV}, {@code OCONV}, {@code COMPLEXPREFIXES}, + * {@code COMPOUNDRULE}, {@code IGNORE}, or {@code KEEPCASE} would change stems + * with no signal. */ @ParameterizedTest @CsvSource({ "ICONV, ICONV 1", "OCONV, OCONV 1", - "COMPLEXPREFIXES, COMPLEXPREFIXES" + "COMPLEXPREFIXES, COMPLEXPREFIXES", + "COMPOUNDRULE, COMPOUNDRULE 1", + "IGNORE, IGNORE x", + "KEEPCASE, KEEPCASE k" }) void testResultAlteringUnsupportedDirectiveFailsLoud(String name, String line) { final IOException e = Assertions.assertThrows(IOException.class, @@ -1248,6 +1252,24 @@ void testResultAlteringUnsupportedDirectiveFailsLoud(String name, String line) { e.getMessage()); } + /** + * Verifies that a full-strip suffix rule is not applied unless the affix file + * declares {@code FULLSTRIP}. Hunspell applies a rule whose strip string consumes + * the whole stem only under that declaration; without it, inventing a stem from + * such a rule contradicts the fail-closed loader policy. + * + * @throws IOException Thrown if a fixture fails to load. + */ + @Test + void testFullStripRuleRequiresFullStripDirective() throws IOException { + final String rule = "SFX A Y 1\nSFX A work ed .\n"; + final String words = "1\nwork/A\n"; + Assertions.assertEquals(List.of("work"), + new HunspellStemmer(load("FULLSTRIP\n" + rule, words)).stemAll("ed")); + Assertions.assertEquals(List.of("ed"), + new HunspellStemmer(load(rule, words)).stemAll("ed")); + } + /** * Verifies that {@link HunspellDictionary#load(InputStream, InputStream)} rejects an * affix stream larger than {@link HunspellDictionary#MAX_STREAM_BYTES}. From 39b576e4563c3b7c8b5a3527d7cc153efc031628 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 08:36:00 -0400 Subject: [PATCH 24/24] OPENNLP-1893: Fail loud on COMPOUNDRULE, IGNORE, KEEPCASE and gate full-strip rules behind FULLSTRIP COMPOUNDRULE licenses pattern compounds, IGNORE drops characters before matching, and KEEPCASE forbids the capitalized variants this stemmer analyzes through lowercasing, so ignoring any of them would change stems with no signal; they now join ICONV, OCONV, and COMPLEXPREFIXES in the load-time rejection. An affix rule whose strip string consumes the whole stem is now undone only when the affix file declares FULLSTRIP; without the declaration the rule is skipped at match time, which is what hunspell itself does rather than rejecting the file. The manual and the dictionary README follow the loader. --- dev/README-hunspell-dictionaries.md | 2 +- .../stemmer/hunspell/HunspellDictionary.java | 28 +++++++++++++++++-- .../stemmer/hunspell/HunspellStemmer.java | 16 ++++++++--- opennlp-docs/src/docbkx/stemmer.xml | 5 +++- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/dev/README-hunspell-dictionaries.md b/dev/README-hunspell-dictionaries.md index b1a1f2f7de..1d125fa131 100644 --- a/dev/README-hunspell-dictionaries.md +++ b/dev/README-hunspell-dictionaries.md @@ -74,4 +74,4 @@ The in-tree tests run against project-authored fixtures only. An opt-in test cla ## What the engine supports -Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, the `AF` flag alias table, the `SET` encoding declaration, compound decomposition under `COMPOUNDFLAG`, the positional `COMPOUNDBEGIN`/`COMPOUNDMIDDLE`/`COMPOUNDEND` flags, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, and the `CHECKCOMPOUNDDUP`/`CHECKCOMPOUNDCASE`/`CHECKCOMPOUNDTRIPLE` declarations (compound parts stand on their entries alone or on an entry plus one affix, the zero and dash suffixes dictionaries position linking forms with included), the blocking flags `NEEDAFFIX` (alias `PSEUDOROOT`), `ONLYINCOMPOUND`, and `FORBIDDENWORD`, which keep virtual stems, compound-only parts, and forbidden words out of the reported analyses, and `CIRCUMFIX`, which binds marked prefix and suffix halves to one another as in the German `ge...t` participle. Directives that would change stems when ignored (`ICONV`, `OCONV`, `COMPLEXPREFIXES`) fail at load time. Cosmetic tables such as `REP`, `MAP`, and `KEY` are skipped, so analyses that would need them are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. Each affix or dictionary stream is rejected when it exceeds `HunspellDictionary.MAX_STREAM_BYTES` (64 MiB). +Supported affix features: `PFX` and `SFX` rules with strip strings, character-class conditions, cross-product combination of one prefix with one suffix, twofold suffixes through continuation classes, `FLAG` modes `char`, `UTF-8`, `long`, and `num`, the `AF` flag alias table, the `SET` encoding declaration, compound decomposition under `COMPOUNDFLAG`, the positional `COMPOUNDBEGIN`/`COMPOUNDMIDDLE`/`COMPOUNDEND` flags, `COMPOUNDMIN`, `COMPOUNDWORDMAX`, `COMPOUNDPERMITFLAG`, `COMPOUNDFORBIDFLAG`, and the `CHECKCOMPOUNDDUP`/`CHECKCOMPOUNDCASE`/`CHECKCOMPOUNDTRIPLE` declarations (compound parts stand on their entries alone or on an entry plus one affix, the zero and dash suffixes dictionaries position linking forms with included), the blocking flags `NEEDAFFIX` (alias `PSEUDOROOT`), `ONLYINCOMPOUND`, and `FORBIDDENWORD`, which keep virtual stems, compound-only parts, and forbidden words out of the reported analyses, and `CIRCUMFIX`, which binds marked prefix and suffix halves to one another as in the German `ge...t` participle, and the `FULLSTRIP` declaration, without which a rule that strips a whole stem is not applied, matching Hunspell. Directives that would change stems when ignored (`ICONV`, `OCONV`, `COMPLEXPREFIXES`, `COMPOUNDRULE`, `IGNORE`, `KEEPCASE`) fail at load time. Cosmetic tables such as `REP`, `MAP`, and `KEY` are skipped, so analyses that would need them are missed rather than invented. A malformed `.aff` file fails loudly at load time with the offending line number in the message. Each affix or dictionary stream is rejected when it exceeds `HunspellDictionary.MAX_STREAM_BYTES` (64 MiB). diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java index 405bcbe46b..f9189838a4 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellDictionary.java @@ -55,9 +55,12 @@ * {@code NEEDAFFIX} (with its historical alias {@code PSEUDOROOT}), * {@code ONLYINCOMPOUND}, and {@code FORBIDDENWORD}, which suppress analyses the * dictionary marks as virtual stems, compound-only parts, or forbidden words; and - * {@code CIRCUMFIX}, which binds marked prefix and suffix halves to one another. + * {@code CIRCUMFIX}, which binds marked prefix and suffix halves to one another; and + * the {@code FULLSTRIP} declaration, without which a rule that strips a whole stem is + * not applied, matching hunspell. * Directives that would change stems when ignored ({@code ICONV}, {@code OCONV}, - * {@code COMPLEXPREFIXES}) are rejected at load time. Cosmetic tables such as + * {@code COMPLEXPREFIXES}, {@code COMPOUNDRULE}, {@code IGNORE}, + * {@code KEEPCASE}) are rejected at load time. Cosmetic tables such as * {@code REP}, {@code MAP}, and {@code KEY} are skipped, so analyses that would need * them are missed rather than invented.

* @@ -148,6 +151,7 @@ enum CompoundPosition { private final boolean checkCompoundDup; private final boolean checkCompoundCase; private final boolean checkCompoundTriple; + private final boolean fullStrip; /** * Initializes the dictionary from the two parsed files. @@ -171,6 +175,7 @@ private HunspellDictionary(Map> entries, AffixFile affix) { this.checkCompoundDup = affix.checkCompoundDup; this.checkCompoundCase = affix.checkCompoundCase; this.checkCompoundTriple = affix.checkCompoundTriple; + this.fullStrip = affix.fullStrip; this.entries = entries; // A material-bearing rule can only be undone from a word whose boundary // character matches its affix material, so bucketing by that character @@ -416,6 +421,11 @@ boolean checkCompoundTriple() { return checkCompoundTriple; } + /** {@return whether {@code FULLSTRIP} allows an affix rule to strip a whole stem} */ + boolean fullStrip() { + return fullStrip; + } + /** * The flag admitting a part at a compound position, next to the general * compounding flag. @@ -722,6 +732,7 @@ private static final class AffixFile { private boolean checkCompoundDup; private boolean checkCompoundCase; private boolean checkCompoundTriple; + private boolean fullStrip; } /** @@ -733,7 +744,8 @@ private static final class AffixFile { * @param content The decoded affix file content. * @return The parsed rules and flag mode. Never {@code null}. * @throws IOException Thrown if a supported directive is malformed, or if - * {@code ICONV}, {@code OCONV}, or {@code COMPLEXPREFIXES} appears. + * {@code ICONV}, {@code OCONV}, {@code COMPLEXPREFIXES}, {@code COMPOUNDRULE}, + * {@code IGNORE}, or {@code KEEPCASE} appears. */ private static AffixFile parseAffix(String content) throws IOException { final AffixFile result = new AffixFile(); @@ -811,6 +823,10 @@ private static AffixFile parseAffix(String content) throws IOException { result.checkCompoundTriple = true; i++; break; + case "FULLSTRIP": + result.fullStrip = true; + i++; + break; case "AF": // the first AF line declares the alias count; every further AF line is one // alias, a flag run whose 1-based position numeric dictionary flags refer to @@ -830,6 +846,12 @@ private static AffixFile parseAffix(String content) throws IOException { case "ICONV": case "OCONV": case "COMPLEXPREFIXES": + // COMPOUNDRULE licenses pattern compounds, IGNORE drops characters before + // matching, and KEEPCASE forbids the case variants this stemmer analyzes; + // ignoring any of them would change stems with no signal + case "COMPOUNDRULE": + case "IGNORE": + case "KEEPCASE": throw new IOException("unsupported affix directive '" + fields[0] + "' at line " + (i + 1)); default: diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java index 4687283518..656064da37 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/hunspell/HunspellStemmer.java @@ -547,7 +547,10 @@ private void undoCrossProductSuffix(String stem, Affix prefix, Affix suffix, * the strip string the rule removed on application, and checks the rule's condition * against the restored stem. A strip-only rule, whose affix material is empty, is * undone by restoring its strip string alone. Rules that neither add nor remove - * material and candidates that would leave an empty stem are rejected. + * material and candidates that would leave an empty stem are rejected. A word the + * affix material covers entirely reverses a full-strip application, which hunspell + * only performs when the affix file declares {@code FULLSTRIP}; without that + * declaration the rule does not apply. * * @param word The surface form. * @param suffix The rule to undo. @@ -557,7 +560,8 @@ private String removeSuffix(String word, Affix suffix) { final String affix = suffix.affix(); final String strip = suffix.strip(); if (affix.isEmpty() && strip.isEmpty() || !word.endsWith(affix) - || word.length() - affix.length() + strip.length() == 0) { + || word.length() - affix.length() + strip.length() == 0 + || (word.length() == affix.length() && !dictionary.fullStrip())) { return null; } final String stem = word.substring(0, word.length() - affix.length()) + strip; @@ -569,7 +573,10 @@ private String removeSuffix(String word, Affix suffix) { * restores the strip string the rule removed on application, and checks the rule's * condition against the restored stem. A strip-only rule, whose affix material is * empty, is undone by restoring its strip string alone. Rules that neither add nor - * remove material and candidates that would leave an empty stem are rejected. + * remove material and candidates that would leave an empty stem are rejected. A + * word the affix material covers entirely reverses a full-strip application, which + * hunspell only performs when the affix file declares {@code FULLSTRIP}; without + * that declaration the rule does not apply. * * @param word The surface form. * @param prefix The rule to undo. @@ -579,7 +586,8 @@ private String removePrefix(String word, Affix prefix) { final String affix = prefix.affix(); final String strip = prefix.strip(); if (affix.isEmpty() && strip.isEmpty() || !word.startsWith(affix) - || word.length() - affix.length() + strip.length() == 0) { + || word.length() - affix.length() + strip.length() == 0 + || (word.length() == affix.length() && !dictionary.fullStrip())) { return null; } final String stem = strip + word.substring(affix.length()); diff --git a/opennlp-docs/src/docbkx/stemmer.xml b/opennlp-docs/src/docbkx/stemmer.xml index 8b02a2f607..711b7c5aaa 100644 --- a/opennlp-docs/src/docbkx/stemmer.xml +++ b/opennlp-docs/src/docbkx/stemmer.xml @@ -98,8 +98,11 @@ stemmer.stem("table"); // "table" (unknown vocabulary is unchanged)]]> (HunspellDictionaryDownload.downloadFromCatalog) needs -Dopennlp.download.remote=true and verifies SHA-512 digests. Directives that would change stems when ignored - (ICONV, OCONV, COMPLEXPREFIXES) + (ICONV, OCONV, COMPLEXPREFIXES, + COMPOUNDRULE, IGNORE, KEEPCASE) fail at load time; cosmetic tables such as REP are skipped. + A rule that strips a whole stem applies only when the affix file + declares FULLSTRIP, as in Hunspell itself. Each affix or dictionary stream is rejected when it exceeds HunspellDictionary.MAX_STREAM_BYTES (64 MiB).