From 595ed48f72742c20f5fabd1daede163239260407 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 16:44:20 -0400 Subject: [PATCH 01/15] OPENNLP-1880: Lexical knowledge base seam with WN-LMF and WNDB readers and a Morphy lemmatizer Adds the LexicalKnowledgeBase contract in opennlp.tools.wordnet and the opennlp-wordnet module implementing it twice: WnLmfReader for WN-LMF XML and WndbReader for the legacy WNDB database files, with reader-equivalence coverage over miniature fixtures of both formats. The WN-LMF reader skips DOCTYPE declarations unresolved with DTD support off, so Open English WordNet releases parse unmodified while entity expansion stays closed; the WNDB fixtures are pinned to LF so their embedded byte offsets survive Windows checkout. The Morphy lemmatizer resolves inflected forms through suffix rules and the format's exception lists. Null arguments fail loudly with IllegalArgumentException, malformed data raises the checked InvalidFormatException, and the public seam carries no brand name: WordNet stays in the names of the classes that actually read WordNet formats. --- .../tools/wordnet/LexicalKnowledgeBase.java | 89 +++ .../java/opennlp/tools/wordnet/Synset.java | 123 ++++ .../opennlp/tools/wordnet/WordNetPOS.java | 40 ++ .../tools/wordnet/WordNetRelation.java | 115 ++++ .../wordnet/LexicalKnowledgeBaseTest.java | 105 +++ .../opennlp/tools/wordnet/SynsetTest.java | 150 +++++ opennlp-distr/pom.xml | 4 + opennlp-distr/src/main/assembly/bin.xml | 7 + opennlp-extensions/opennlp-wordnet/pom.xml | 59 ++ .../wordnet/InMemoryWordNetLexicon.java | 156 +++++ .../java/opennlp/wordnet/LemmaFolding.java | 71 ++ .../opennlp/wordnet/MorphyExceptions.java | 145 ++++ .../opennlp/wordnet/MorphyLemmatizer.java | 226 +++++++ .../java/opennlp/wordnet/WnLmfReader.java | 600 +++++++++++++++++ .../main/java/opennlp/wordnet/WndbReader.java | 627 ++++++++++++++++++ .../wordnet/InMemoryWordNetLexiconTest.java | 89 +++ .../opennlp/wordnet/LemmaFoldingTest.java | 66 ++ .../wordnet/LexiconConcurrencyTest.java | 91 +++ .../opennlp/wordnet/MorphyExceptionsTest.java | 124 ++++ .../opennlp/wordnet/MorphyLemmatizerTest.java | 188 ++++++ .../wordnet/ReaderEquivalenceTest.java | 121 ++++ .../java/opennlp/wordnet/WnLmfReaderTest.java | 405 +++++++++++ .../java/opennlp/wordnet/WndbReaderTest.java | 273 ++++++++ .../resources/opennlp/wordnet/mini-wn-lmf.xml | 183 +++++ .../opennlp/wordnet/mini-wndb/.gitattributes | 7 + .../opennlp/wordnet/mini-wndb/adj.exc | 1 + .../opennlp/wordnet/mini-wndb/adv.exc | 1 + .../opennlp/wordnet/mini-wndb/data.adj | 22 + .../opennlp/wordnet/mini-wndb/data.adv | 20 + .../opennlp/wordnet/mini-wndb/data.noun | 27 + .../opennlp/wordnet/mini-wndb/data.verb | 22 + .../opennlp/wordnet/mini-wndb/index.adj | 22 + .../opennlp/wordnet/mini-wndb/index.adv | 20 + .../opennlp/wordnet/mini-wndb/index.noun | 27 + .../opennlp/wordnet/mini-wndb/index.verb | 22 + .../opennlp/wordnet/mini-wndb/noun.exc | 3 + .../opennlp/wordnet/mini-wndb/verb.exc | 4 + opennlp-extensions/pom.xml | 1 + pom.xml | 6 + rat-excludes | 9 + 40 files changed, 4271 insertions(+) create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/pom.xml create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java new file mode 100644 index 0000000000..214ef74db1 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.List; +import java.util.Optional; + +/** + * Lemma and synset lookup over a loaded lexical-semantic resource in the WordNet family. Synset + * identity is opaque and source-qualified (see {@link Synset#id()}). Lookups return their matches + * in the source's sense order and never return {@code null}. + * + *

Lemma matching semantics are the implementation's concern. The reference implementations + * match case-insensitively (case folding with the root locale) and treat the underscore some + * formats store in multiword lemmas as a space; an implementation with different semantics must + * document them. Returned {@link Synset#lemmas() lemmas} preserve the source's written forms, + * with spaces in multiword lemmas.

+ * + *

Implementations must be immutable and thread-safe after loading: one instance is meant to + * be shared across an application's threads for concurrent lookups.

+ */ +public interface LexicalKnowledgeBase { + + /** + * Finds the synsets containing a lemma with a part of speech, in the source's sense order + * (the most salient sense first when the source ranks senses). + * + * @param lemma The lemma to look up. Must not be {@code null}. + * @param pos The part of speech to look it up as. Must not be {@code null}. + * @return The matching synsets, never {@code null}; empty when the lexicon does not contain + * the lemma with that part of speech. + * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}. + */ + List lookup(String lemma, WordNetPOS pos); + + /** + * Finds a synset by its opaque identifier. + * + * @param synsetId The synset identifier, as minted by this lexicon. Must not be {@code null}. + * @return The synset, or empty when this lexicon has no synset with that identifier. + * @throws IllegalArgumentException Thrown if {@code synsetId} is {@code null}. + */ + Optional synset(String synsetId); + + /** + * Navigates one typed relation from a synset. + * + * @param synsetId The source synset identifier. Must not be {@code null}. + * @param relation The relation type to follow. Must not be {@code null}. + * @return The target synset ids in source order, never {@code null}; empty when the synset is + * unknown or has no relation of that type. + * @throws IllegalArgumentException Thrown if {@code synsetId} or {@code relation} is + * {@code null}. + */ + default List related(String synsetId, WordNetRelation relation) { + if (relation == null) { + throw new IllegalArgumentException("Relation must not be null"); + } + return synset(synsetId).map(s -> s.related(relation)).orElse(List.of()); + } + + /** + * Tests whether the lexicon contains a lemma with a part of speech. This is the membership + * check morphological rules validate their candidates against; implementations may override + * it with a cheaper check than {@link #lookup(String, WordNetPOS)}. + * + * @param lemma The lemma to test. Must not be {@code null}. + * @param pos The part of speech to test it as. Must not be {@code null}. + * @return {@code true} if the lexicon contains the lemma with that part of speech. + * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}. + */ + default boolean contains(String lemma, WordNetPOS pos) { + return !lookup(lemma, pos).isEmpty(); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java new file mode 100644 index 0000000000..477fa38d90 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; + +/** + * One synonym set: a single lexicalized concept with its member lemmas, gloss, and typed + * relations to other synsets. + * + *

The {@link #id() id} is an opaque, source-qualified string minted by the reader that + * produced the synset; consumers must not parse it, only pass it back to + * {@link LexicalKnowledgeBase#synset(String)} and compare it for equality. Relations map each + * {@link WordNetRelation} present on this synset to the target synset ids in source order.

+ * + *

Instances are immutable and thread-safe: the list and map components are defensively + * copied to immutable views at construction.

+ * + * @param id The opaque, source-qualified synset identifier. Must not be {@code null} or + * empty. + * @param pos The part of speech. Must not be {@code null}. + * @param lemmas The member lemmas in source order, human-readable (multiword lemmas use + * spaces, not the underscores some formats store). Must not be {@code null} or + * empty, and must not contain {@code null} or empty elements. + * @param gloss The definition text, possibly empty when the source has none. Must not be + * {@code null}. + * @param relations The typed relations, each mapping to the target synset ids in source order. + * Must not be {@code null}; keys must not be {@code null}; each value must be + * a non-empty list of non-{@code null}, non-empty target ids. + */ +@ThreadSafe +public record Synset( + String id, + WordNetPOS pos, + List lemmas, + String gloss, + Map> relations) { + + /** + * Creates a synset. + * + * @throws IllegalArgumentException Thrown if any component violates its documented constraint. + */ + public Synset { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("Id must not be null or empty"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + if (lemmas == null || lemmas.isEmpty()) { + throw new IllegalArgumentException("Lemmas must not be null or empty for synset " + id); + } + for (final String lemma : lemmas) { + if (lemma == null || lemma.isEmpty()) { + throw new IllegalArgumentException( + "Lemmas must not contain a null or empty element for synset " + id); + } + } + if (gloss == null) { + throw new IllegalArgumentException("Gloss must not be null for synset " + id); + } + if (relations == null) { + throw new IllegalArgumentException("Relations must not be null for synset " + id); + } + final Map> copiedRelations = + new EnumMap<>(WordNetRelation.class); + for (final Map.Entry> relation : relations.entrySet()) { + if (relation.getKey() == null) { + throw new IllegalArgumentException("Relations must not contain a null key for synset " + id); + } + final List targets = relation.getValue(); + if (targets == null || targets.isEmpty()) { + throw new IllegalArgumentException("Relation " + relation.getKey() + + " must map to a non-empty target list for synset " + id); + } + for (final String target : targets) { + if (target == null || target.isEmpty()) { + throw new IllegalArgumentException("Relation " + relation.getKey() + + " must not contain a null or empty target id for synset " + id); + } + } + copiedRelations.put(relation.getKey(), List.copyOf(targets)); + } + lemmas = List.copyOf(lemmas); + relations = Collections.unmodifiableMap(copiedRelations); + } + + /** + * Finds the target synset ids of one relation type. + * + * @param relation The relation type. Must not be {@code null}. + * @return The target synset ids in source order, never {@code null}; empty when this synset + * has no relation of that type. + * @throws IllegalArgumentException Thrown if {@code relation} is {@code null}. + */ + public List related(WordNetRelation relation) { + if (relation == null) { + throw new IllegalArgumentException("Relation must not be null"); + } + final List targets = relations.get(relation); + return targets == null ? List.of() : targets; + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java new file mode 100644 index 0000000000..65f6e691ec --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +/** + * The four parts of speech a wordnet-style lexicon distinguishes. + * + *

The enum carries none of the single-letter codes the on-disk formats use; readers own the + * mapping from their format's codes to these values. Adjective satellites normalize to + * {@link #ADJECTIVE}, with the cluster structure preserved through + * {@link WordNetRelation#SIMILAR_TO}.

+ */ +public enum WordNetPOS { + + /** Nouns. */ + NOUN, + + /** Verbs. */ + VERB, + + /** Adjectives, including adjective satellites. */ + ADJECTIVE, + + /** Adverbs. */ + ADVERB +} diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java new file mode 100644 index 0000000000..10da3cea34 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +/** + * The typed relations a wordnet-style lexicon draws between {@link Synset synsets}. Readers map + * their source format's relation names onto these values. + * + *

Relations that a source format draws between individual word senses (antonymy and + * derivation, for example) surface here at the synset level: the synset containing the source + * sense carries the relation to the synset containing the target sense.

+ */ +public enum WordNetRelation { + + /** Opposition in meaning, for example between the adjectives for tall and short. */ + ANTONYM, + + /** The more general concept: a dog is a kind of canid. */ + HYPERNYM, + + /** The class a named instance belongs to: a specific river is an instance of river. */ + INSTANCE_HYPERNYM, + + /** The more specific concept: canid has the hyponym dog. */ + HYPONYM, + + /** A named instance of this class. */ + INSTANCE_HYPONYM, + + /** The group this synset is a member of. */ + MEMBER_HOLONYM, + + /** The whole this synset is a substance of. */ + SUBSTANCE_HOLONYM, + + /** The whole this synset is a part of. */ + PART_HOLONYM, + + /** A member of this group. */ + MEMBER_MERONYM, + + /** A substance this synset is made of. */ + SUBSTANCE_MERONYM, + + /** A part of this synset. */ + PART_MERONYM, + + /** The attribute a value expresses, or a value of this attribute. */ + ATTRIBUTE, + + /** A derivationally related form, typically across parts of speech. */ + DERIVATIONALLY_RELATED, + + /** An action entailed by this verb: snoring entails sleeping. */ + ENTAILMENT, + + /** The verb that entails this one; the inverse of {@link #ENTAILMENT}. */ + ENTAILED_BY, + + /** An effect this verb causes. */ + CAUSE, + + /** The cause of this verb; the inverse of {@link #CAUSE}. */ + CAUSED_BY, + + /** A related synset worth consulting. */ + ALSO_SEE, + + /** A verb sense grouped with this one. */ + VERB_GROUP, + + /** A satellite or head adjective in the same similarity cluster. */ + SIMILAR_TO, + + /** The verb an adjective is the participle of. */ + PARTICIPLE, + + /** + * The noun an adjective pertains to, or the adjective an adverb derives from. The source + * formats use one pointer for both directions of derivation, so this value does too. + */ + PERTAINYM, + + /** The topical domain this synset belongs to. */ + DOMAIN_TOPIC, + + /** A synset belonging to this topical domain. */ + MEMBER_OF_DOMAIN_TOPIC, + + /** The regional domain this synset belongs to. */ + DOMAIN_REGION, + + /** A synset belonging to this regional domain. */ + MEMBER_OF_DOMAIN_REGION, + + /** The usage domain this synset belongs to, for example slang or archaism. */ + DOMAIN_USAGE, + + /** A synset belonging to this usage domain. */ + MEMBER_OF_DOMAIN_USAGE +} diff --git a/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java b/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java new file mode 100644 index 0000000000..2b698797f1 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the {@link LexicalKnowledgeBase} default methods against a minimal in-memory + * implementation, so the defaults are validated independently of any reader. + */ +public class LexicalKnowledgeBaseTest { + + private static final Synset DOG = new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), + "a domesticated canid", Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n"))); + + private static final Synset CANID = new Synset("test-2-n", WordNetPOS.NOUN, List.of("canid"), + "a carnivorous mammal", Map.of(WordNetRelation.HYPONYM, List.of("test-1-n"))); + + // A deliberately tiny implementation of only the two abstract methods. + private static final LexicalKnowledgeBase LEXICON = new LexicalKnowledgeBase() { + + @Override + public List lookup(String lemma, WordNetPOS pos) { + if (lemma == null) { + throw new IllegalArgumentException("Lemma must not be null"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + if (pos == WordNetPOS.NOUN && "dog".equals(lemma)) { + return List.of(DOG); + } + return List.of(); + } + + @Override + public Optional synset(String synsetId) { + if (synsetId == null) { + throw new IllegalArgumentException("SynsetId must not be null"); + } + if (DOG.id().equals(synsetId)) { + return Optional.of(DOG); + } + if (CANID.id().equals(synsetId)) { + return Optional.of(CANID); + } + return Optional.empty(); + } + }; + + @Test + void testRelatedNavigatesThroughSynset() { + assertEquals(List.of("test-2-n"), LEXICON.related("test-1-n", WordNetRelation.HYPERNYM)); + assertEquals(List.of("test-1-n"), LEXICON.related("test-2-n", WordNetRelation.HYPONYM)); + } + + @Test + void testRelatedIsEmptyForAbsentRelationOrUnknownSynset() { + assertTrue(LEXICON.related("test-1-n", WordNetRelation.ANTONYM).isEmpty()); + assertTrue(LEXICON.related("test-99-n", WordNetRelation.HYPERNYM).isEmpty()); + } + + @Test + void testRelatedRejectsNulls() { + assertThrows(IllegalArgumentException.class, + () -> LEXICON.related(null, WordNetRelation.HYPERNYM)); + assertThrows(IllegalArgumentException.class, () -> LEXICON.related("test-1-n", null)); + } + + @Test + void testContainsFollowsLookup() { + assertTrue(LEXICON.contains("dog", WordNetPOS.NOUN)); + assertFalse(LEXICON.contains("dog", WordNetPOS.VERB)); + assertFalse(LEXICON.contains("cat", WordNetPOS.NOUN)); + } + + @Test + void testContainsRejectsNulls() { + assertThrows(IllegalArgumentException.class, () -> LEXICON.contains(null, WordNetPOS.NOUN)); + assertThrows(IllegalArgumentException.class, () -> LEXICON.contains("dog", null)); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java b/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java new file mode 100644 index 0000000000..e9cbb4c063 --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SynsetTest { + + private static Synset dog() { + return new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog", "domestic dog"), + "a domesticated canid", + Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n"))); + } + + @Test + void testComponents() { + final Synset synset = dog(); + assertEquals("test-1-n", synset.id()); + assertEquals(WordNetPOS.NOUN, synset.pos()); + assertEquals(List.of("dog", "domestic dog"), synset.lemmas()); + assertEquals("a domesticated canid", synset.gloss()); + assertEquals(Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n")), synset.relations()); + } + + @Test + void testRelatedReturnsTargetsInOrder() { + final Synset synset = new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", + Map.of(WordNetRelation.HYPONYM, List.of("test-3-n", "test-2-n"))); + assertEquals(List.of("test-3-n", "test-2-n"), synset.related(WordNetRelation.HYPONYM)); + } + + @Test + void testRelatedIsEmptyForAbsentRelation() { + assertTrue(dog().related(WordNetRelation.ANTONYM).isEmpty()); + } + + @Test + void testRelatedRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> dog().related(null)); + } + + @Test + void testEmptyGlossAndNoRelationsAreValid() { + final Synset synset = new Synset("test-9-r", WordNetPOS.ADVERB, List.of("well"), "", Map.of()); + assertEquals("", synset.gloss()); + assertTrue(synset.relations().isEmpty()); + } + + @Test + void testDefensiveCopies() { + final List lemmas = new ArrayList<>(List.of("dog")); + final List targets = new ArrayList<>(List.of("test-2-n")); + final Map> relations = new HashMap<>(); + relations.put(WordNetRelation.HYPERNYM, targets); + final Synset synset = new Synset("test-1-n", WordNetPOS.NOUN, lemmas, "gloss", relations); + lemmas.add("mutated"); + targets.add("mutated"); + relations.put(WordNetRelation.ANTONYM, List.of("test-3-n")); + assertEquals(List.of("dog"), synset.lemmas()); + assertEquals(List.of("test-2-n"), synset.related(WordNetRelation.HYPERNYM)); + assertEquals(1, synset.relations().size()); + } + + @Test + void testReturnedCollectionsAreImmutable() { + final Synset synset = dog(); + assertThrows(UnsupportedOperationException.class, () -> synset.lemmas().add("x")); + assertThrows(UnsupportedOperationException.class, + () -> synset.relations().put(WordNetRelation.ANTONYM, List.of("x"))); + assertThrows(UnsupportedOperationException.class, + () -> synset.related(WordNetRelation.HYPERNYM).add("x")); + } + + @Test + void testRejectsNullOrEmptyId() { + assertThrows(IllegalArgumentException.class, + () -> new Synset(null, WordNetPOS.NOUN, List.of("dog"), "", Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new Synset("", WordNetPOS.NOUN, List.of("dog"), "", Map.of())); + } + + @Test + void testRejectsNullPos() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", null, List.of("dog"), "", Map.of())); + } + + @Test + void testRejectsNullOrEmptyLemmas() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, null, "", Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of(), "", Map.of())); + final List withNull = new ArrayList<>(); + withNull.add(null); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, withNull, "", Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of(""), "", Map.of())); + } + + @Test + void testRejectsNullGloss() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), null, Map.of())); + } + + @Test + void testRejectsInvalidRelations() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", null)); + final Map> nullKey = new HashMap<>(); + nullKey.put(null, List.of("test-2-n")); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", nullKey)); + final Map> nullTargets = new HashMap<>(); + nullTargets.put(WordNetRelation.HYPERNYM, null); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", nullTargets)); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", + Map.of(WordNetRelation.HYPERNYM, List.of()))); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", + Map.of(WordNetRelation.HYPERNYM, List.of("")))); + } +} diff --git a/opennlp-distr/pom.xml b/opennlp-distr/pom.xml index e9092d8821..39269575e8 100644 --- a/opennlp-distr/pom.xml +++ b/opennlp-distr/pom.xml @@ -91,6 +91,10 @@ org.apache.opennlp opennlp-spellcheck + + org.apache.opennlp + opennlp-wordnet + diff --git a/opennlp-distr/src/main/assembly/bin.xml b/opennlp-distr/src/main/assembly/bin.xml index 2db4eafc65..36c556d473 100644 --- a/opennlp-distr/src/main/assembly/bin.xml +++ b/opennlp-distr/src/main/assembly/bin.xml @@ -239,6 +239,13 @@ docs/apidocs/opennlp-spellcheck + + ../opennlp-extensions/opennlp-wordnet/target/reports/apidocs + 644 + 755 + docs/apidocs/opennlp-wordnet + + ../opennlp-extensions/opennlp-uima/target/reports/apidocs 644 diff --git a/opennlp-extensions/opennlp-wordnet/pom.xml b/opennlp-extensions/opennlp-wordnet/pom.xml new file mode 100644 index 0000000000..0721fcfc98 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/pom.xml @@ -0,0 +1,59 @@ + + + + + + 4.0.0 + + org.apache.opennlp + opennlp-extensions + 3.0.0-SNAPSHOT + + + opennlp-wordnet + jar + Apache OpenNLP :: Ext :: WordNet + + + + org.apache.opennlp + opennlp-api + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java new file mode 100644 index 0000000000..2401bfab6a --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * The immutable in-memory {@link LexicalKnowledgeBase} both readers produce: a synset table plus a + * folded (lemma, part of speech) index. Package-private because it is a reader product, not a + * public entry point; consumers hold it as {@link LexicalKnowledgeBase}. + * + *

Keys and queries are folded identically (see {@link LemmaFolding}). Construction verifies + * referential integrity: every relation target of every synset must resolve to a synset in the + * table, so a lexicon can never hand out a dangling identifier. After construction all state is + * immutable, making instances safe for concurrent lookups.

+ */ +@ThreadSafe +final class InMemoryWordNetLexicon implements LexicalKnowledgeBase { + + private final Map synsetsById; + private final Map> senseIndex; + + /** + * Indexes the given synsets. + * + * @param synsetsById The synset table keyed by synset id; every key must equal its synset's + * {@link Synset#id() id}. Must not be {@code null}. + * @param senseOrder The sense order per folded (lemma, part of speech) key: for each key the + * ids of the synsets containing the lemma, most salient sense first, each + * id resolvable in {@code synsetsById} and free of duplicates per key. + * Must not be {@code null}. + * @throws IllegalArgumentException Thrown if a relation target or sense entry does not + * resolve, or a key disagrees with its synset id. + */ + InMemoryWordNetLexicon(Map synsetsById, Map> senseOrder) { + if (synsetsById == null) { + throw new IllegalArgumentException("SynsetsById must not be null"); + } + if (senseOrder == null) { + throw new IllegalArgumentException("SenseOrder must not be null"); + } + final Map byId = new HashMap<>(synsetsById.size() * 2); + for (final Map.Entry entry : synsetsById.entrySet()) { + if (entry.getValue() == null || !entry.getValue().id().equals(entry.getKey())) { + throw new IllegalArgumentException( + "Synset table key " + entry.getKey() + " does not match its synset"); + } + byId.put(entry.getKey(), entry.getValue()); + } + for (final Synset synset : byId.values()) { + for (final Map.Entry> relation : + synset.relations().entrySet()) { + for (final String target : relation.getValue()) { + if (!byId.containsKey(target)) { + throw new IllegalArgumentException("Synset " + synset.id() + " has a " + + relation.getKey() + " relation to unknown synset " + target); + } + } + } + } + final Map> index = new HashMap<>(senseOrder.size() * 2); + for (final Map.Entry> entry : senseOrder.entrySet()) { + final List senses = new ArrayList<>(entry.getValue().size()); + for (final String synsetId : entry.getValue()) { + final Synset synset = byId.get(synsetId); + if (synset == null) { + throw new IllegalArgumentException("Sense index entry " + entry.getKey().lemma() + + " (" + entry.getKey().pos() + ") references unknown synset " + synsetId); + } + senses.add(synset); + } + index.put(entry.getKey(), List.copyOf(senses)); + } + this.synsetsById = byId; + this.senseIndex = index; + } + + /** {@inheritDoc} */ + @Override + public List lookup(String lemma, WordNetPOS pos) { + if (lemma == null) { + throw new IllegalArgumentException("Lemma must not be null"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + final List senses = senseIndex.get(LemmaKey.of(lemma, pos)); + return senses == null ? List.of() : senses; + } + + /** {@inheritDoc} */ + @Override + public Optional synset(String synsetId) { + if (synsetId == null) { + throw new IllegalArgumentException("SynsetId must not be null"); + } + return Optional.ofNullable(synsetsById.get(synsetId)); + } + + /** {@return the number of synsets in this lexicon} */ + int size() { + return synsetsById.size(); + } + + /** {@return all synsets, for equivalence checks and diagnostics within this package} */ + Collection synsets() { + return Collections.unmodifiableCollection(synsetsById.values()); + } + + /** + * A folded sense-index key. Build with {@link #of(String, WordNetPOS)} so every key passes + * through the same fold as every query. + * + * @param lemma The folded lemma. + * @param pos The part of speech. + */ + record LemmaKey(String lemma, WordNetPOS pos) { + + /** + * Folds a written form into a key: lowercase with the root locale, underscore as space. + * + * @param writtenForm The lemma as written in the source or query. Must not be {@code null}. + * @param pos The part of speech. Must not be {@code null}. + * @return The folded key. + */ + static LemmaKey of(String writtenForm, WordNetPOS pos) { + return new LemmaKey(LemmaFolding.fold(writtenForm), pos); + } + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java new file mode 100644 index 0000000000..4da713d520 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * The single home of the lemma fold and the space-separated field split this package relies on. + * {@link MorphyExceptions} keys, the {@link InMemoryWordNetLexicon.LemmaKey sense-index keys}, + * and every query must fold through {@link #fold(String)} so their canonical forms agree. + */ +final class LemmaFolding { + + /** Not instantiable. */ + private LemmaFolding() { + } + + /** + * Folds a written form into its canonical shape: lowercase with the root locale, with the + * underscore some formats store in multiword lemmas treated as a space. + * + * @param writtenForm The form as written in a source file or query. Must not be {@code null}. + * @return The folded form. + * @throws IllegalArgumentException Thrown if {@code writtenForm} is {@code null}. + */ + static String fold(String writtenForm) { + if (writtenForm == null) { + throw new IllegalArgumentException("writtenForm must not be null"); + } + return writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT); + } + + /** + * Splits a space-separated field list, collapsing runs of spaces. + * + * @param value The field list. Must not be {@code null}. + * @return The non-empty fields in order, never {@code null}. + */ + static List splitOnSpaces(String value) { + final List parts = new ArrayList<>(4); + int start = 0; + while (start < value.length()) { + final int space = value.indexOf(' ', start); + if (space < 0) { + parts.add(value.substring(start)); + break; + } + if (space > start) { + parts.add(value.substring(start, space)); + } + start = space + 1; + } + return parts; + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java new file mode 100644 index 0000000000..4f6c0b2934 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.WordNetPOS; + +/** + * The Morphy exception lists: the per-part-of-speech tables of irregular inflected forms + * ({@code mice} to {@code mouse}, {@code went} to {@code go}) that the Morphy algorithm + * consults before its detachment rules. + * + *

{@link #load(Path)} reads the four {@code *.exc} files ({@code noun.exc}, + * {@code verb.exc}, {@code adj.exc}, {@code adv.exc}), which must all be present, in the WNDB + * format: one entry per line, the inflected form followed by one or more base forms, space + * separated, with underscores standing for spaces in multiword entries. No exception data is + * bundled; the caller supplies a directory.

+ * + *

Lookups fold the queried word the same way the lexicon seam folds lemmas. Instances are + * immutable after loading and safe for concurrent lookups.

+ */ +@ThreadSafe +public final class MorphyExceptions { + + private final Map>> byPos; + + /** + * Wraps the per-part-of-speech exception tables. + * + * @param byPos The loaded tables, one per part of speech. + */ + private MorphyExceptions(Map>> byPos) { + this.byPos = byPos; + } + + /** + * Loads the four exception lists from a directory. + * + * @param directory The directory containing {@code noun.exc}, {@code verb.exc}, + * {@code adj.exc}, and {@code adv.exc}. Must not be {@code null} and must + * exist. + * @return The loaded exception lists. + * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a + * directory. + * @throws InvalidFormatException Thrown if one of the four files is missing or a line is + * malformed; the message names the file and line. + * @throws IOException Thrown if reading a file fails. + */ + public static MorphyExceptions load(Path directory) throws IOException { + if (directory == null) { + throw new IllegalArgumentException("Directory must not be null"); + } + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException( + "Directory does not exist or is not a directory: " + directory); + } + final Map>> byPos = new EnumMap<>(WordNetPOS.class); + byPos.put(WordNetPOS.NOUN, loadFile(directory, "noun.exc")); + byPos.put(WordNetPOS.VERB, loadFile(directory, "verb.exc")); + byPos.put(WordNetPOS.ADJECTIVE, loadFile(directory, "adj.exc")); + byPos.put(WordNetPOS.ADVERB, loadFile(directory, "adv.exc")); + return new MorphyExceptions(byPos); + } + + /** + * Finds the base forms of an irregular inflected form. + * + * @param word The inflected form; folded before lookup. Must not be {@code null}. + * @param pos The part of speech. Must not be {@code null}. + * @return The base forms in file order, never {@code null}; empty when the word has no entry. + * @throws IllegalArgumentException Thrown if {@code word} or {@code pos} is {@code null}. + */ + public List lookup(String word, WordNetPOS pos) { + if (word == null) { + throw new IllegalArgumentException("Word must not be null"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + final List lemmas = byPos.get(pos).get(LemmaFolding.fold(word)); + return lemmas == null ? List.of() : lemmas; + } + + /** + * Loads one {@code *.exc} file into a folded inflected-form to base-forms map. + * + * @param directory The directory holding the file. + * @param fileName The exception file name, for example {@code noun.exc}. + * @return The folded exception entries. + * @throws InvalidFormatException Thrown if the file is missing or a line is malformed. + * @throws IOException Thrown if reading the file fails. + */ + private static Map> loadFile(Path directory, String fileName) + throws IOException { + final Path file = directory.resolve(fileName); + if (!Files.isRegularFile(file)) { + throw new InvalidFormatException("Missing exception list file: " + file); + } + final List lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1); + final Map> entries = new HashMap<>(lines.size() * 2); + for (int i = 0; i < lines.size(); i++) { + final String line = lines.get(i); + if (line.isEmpty()) { + continue; + } + final List fields = LemmaFolding.splitOnSpaces(line); + if (fields.size() < 2) { + throw new InvalidFormatException("Malformed exception list " + fileName + " at line " + + (i + 1) + ": expected an inflected form and at least one base form, got: " + line); + } + final List lemmas = new ArrayList<>(fields.size() - 1); + for (final String lemma : fields.subList(1, fields.size())) { + lemmas.add(LemmaFolding.fold(lemma)); + } + // A form listed twice keeps its first entry, matching first-match lookup semantics. + entries.putIfAbsent(LemmaFolding.fold(fields.get(0)), List.copyOf(lemmas)); + } + return Map.copyOf(entries); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java new file mode 100644 index 0000000000..c4da01d94f --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.lemmatizer.Lemmatizer; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.WordNetPOS; + +/** + * A {@link Lemmatizer} implementing the Morphy algorithm: exception-list lookup first, then the + * per-part-of-speech iterative detachment rules, with every rule-derived candidate validated + * against a {@link LexicalKnowledgeBase} before it is returned. A token is folded (lowercase with + * the root locale, underscore as space) before lookup, and returned lemmas are in that folded + * form. + * + *

Part-of-speech tags map to a {@link WordNetPOS} by their conventional Penn Treebank + * prefixes ({@code N}, {@code V}, {@code J}, {@code R}), the names {@code ADJ} and {@code ADV}, + * and the one-letter WordNet codes {@code n}, {@code v}, {@code a}, {@code r}, and {@code s} + * (satellite, treated as adjective), case-insensitively. A tag that maps to no part of speech + * yields the unknown-word result.

+ * + *

Following {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}, a token with no lemma + * yields {@link #UNKNOWN_LEMMA} from {@link #lemmatize(String[], String[])} and a singleton list + * of it from {@link #lemmatize(List, List)}. Both a lexicon and exception lists are required. + * Instances are immutable and safe for concurrent use.

+ */ +@ThreadSafe +public final class MorphyLemmatizer implements Lemmatizer { + + /** + * The output emitted for a token whose lemma is unknown, following the conventional + * {@link Lemmatizer} unknown marker also used by + * {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}. + */ + public static final String UNKNOWN_LEMMA = "O"; + + private static final String[][] NOUN_RULES = { + {"s", ""}, {"ses", "s"}, {"xes", "x"}, {"zes", "z"}, + {"ches", "ch"}, {"shes", "sh"}, {"men", "man"}, {"ies", "y"}, + }; + + private static final String[][] VERB_RULES = { + {"s", ""}, {"ies", "y"}, {"es", "e"}, {"es", ""}, + {"ed", "e"}, {"ed", ""}, {"ing", "e"}, {"ing", ""}, + }; + + private static final String[][] ADJECTIVE_RULES = { + {"er", ""}, {"est", ""}, {"er", "e"}, {"est", "e"}, + }; + + private static final String[][] NO_RULES = {}; + + private final LexicalKnowledgeBase lexicon; + private final MorphyExceptions exceptions; + + /** + * Creates a Morphy lemmatizer over a loaded lexicon and exception lists. + * + * @param lexicon The lexicon rule candidates are validated against. Must not be + * {@code null}. + * @param exceptions The irregular-form exception lists. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code lexicon} or {@code exceptions} is + * {@code null}. + */ + public MorphyLemmatizer(LexicalKnowledgeBase lexicon, MorphyExceptions exceptions) { + if (lexicon == null) { + throw new IllegalArgumentException("Lexicon must not be null"); + } + if (exceptions == null) { + throw new IllegalArgumentException("Exceptions must not be null"); + } + this.lexicon = lexicon; + this.exceptions = exceptions; + } + + /** + * {@inheritDoc} + * + * @throws IllegalArgumentException Thrown if {@code toks} or {@code tags} is {@code null}, + * contains a {@code null} element, or the two differ in length. + */ + @Override + public String[] lemmatize(String[] toks, String[] tags) { + if (toks == null || tags == null) { + throw new IllegalArgumentException("Toks and tags must not be null"); + } + if (toks.length != tags.length) { + throw new IllegalArgumentException("Toks and tags must have the same length, got " + + toks.length + " and " + tags.length); + } + final String[] lemmas = new String[toks.length]; + for (int i = 0; i < toks.length; i++) { + final List candidates = lemmasOf(toks[i], tags[i]); + lemmas[i] = candidates.isEmpty() ? UNKNOWN_LEMMA : candidates.get(0); + } + return lemmas; + } + + /** + * {@inheritDoc} + * + * @throws IllegalArgumentException Thrown if {@code toks} or {@code tags} is {@code null}, + * contains a {@code null} element, or the two differ in size. + */ + @Override + public List> lemmatize(List toks, List tags) { + if (toks == null || tags == null) { + throw new IllegalArgumentException("Toks and tags must not be null"); + } + if (toks.size() != tags.size()) { + throw new IllegalArgumentException("Toks and tags must have the same size, got " + + toks.size() + " and " + tags.size()); + } + final List> lemmas = new ArrayList<>(toks.size()); + for (int i = 0; i < toks.size(); i++) { + final List candidates = lemmasOf(toks.get(i), tags.get(i)); + lemmas.add(candidates.isEmpty() ? List.of(UNKNOWN_LEMMA) : candidates); + } + return lemmas; + } + + /** + * Finds all lemmas of one token, most preferred first. + * + * @param token The token to lemmatize. + * @param tag The part-of-speech tag. + * @return The candidate lemmas, empty when the word is unknown or the tag maps to no part of + * speech. + */ + private List lemmasOf(String token, String tag) { + if (token == null || tag == null) { + throw new IllegalArgumentException("Tokens and tags must not contain null elements"); + } + final WordNetPOS pos = posFromTag(tag); + if (pos == null) { + return List.of(); + } + final String folded = LemmaFolding.fold(token); + final List irregular = exceptions.lookup(folded, pos); + if (!irregular.isEmpty()) { + return irregular; + } + final List candidates = new ArrayList<>(2); + if (lexicon.contains(folded, pos)) { + candidates.add(folded); + } + for (final String[] rule : rulesFor(pos)) { + final String suffix = rule[0]; + if (folded.length() > suffix.length() && folded.endsWith(suffix)) { + final String candidate = + folded.substring(0, folded.length() - suffix.length()) + rule[1]; + if (!candidates.contains(candidate) && lexicon.contains(candidate, pos)) { + candidates.add(candidate); + } + } + } + return candidates; + } + + /** + * Selects the detachment-rule table for a part of speech. + * + * @param pos The part of speech. + * @return The suffix-substitution rules, empty for adverbs. + */ + private static String[][] rulesFor(WordNetPOS pos) { + return switch (pos) { + case NOUN -> NOUN_RULES; + case VERB -> VERB_RULES; + case ADJECTIVE -> ADJECTIVE_RULES; + case ADVERB -> NO_RULES; + }; + } + + /** + * Maps a part-of-speech tag to a {@link WordNetPOS}. Package-private so tests can pin the + * mapping directly. + * + * @param tag The tag to map. Must not be {@code null}. + * @return The part of speech, or {@code null} when the tag names none. + * @throws IllegalArgumentException Thrown if {@code tag} is {@code null}. + */ + static WordNetPOS posFromTag(String tag) { + if (tag == null) { + throw new IllegalArgumentException("Tag must not be null"); + } + if (tag.isEmpty()) { + return null; + } + final String upper = tag.toUpperCase(Locale.ROOT); + if (upper.startsWith("ADJ")) { + return WordNetPOS.ADJECTIVE; + } + if (upper.startsWith("ADV")) { + return WordNetPOS.ADVERB; + } + return switch (upper.charAt(0)) { + case 'N' -> WordNetPOS.NOUN; + case 'V' -> WordNetPOS.VERB; + case 'J' -> WordNetPOS.ADJECTIVE; + // Codes a and s mean adjective only as one-letter tags; AUX, ADP and the like do not. + case 'A', 'S' -> tag.length() == 1 ? WordNetPOS.ADJECTIVE : null; + case 'R' -> WordNetPOS.ADVERB; + default -> null; + }; + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java new file mode 100644 index 0000000000..1222cfbfd1 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -0,0 +1,600 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.xml.XMLConstants; +import javax.xml.stream.Location; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Reads a WN-LMF XML document (the Global WordNet Association + * interchange format, used by + * Open English WordNet and many + * other language wordnets) into a {@link LexicalKnowledgeBase} using the JDK StAX parser. + * + *

It reads the subset of the format the contract serves: lexical entries, synsets with their + * definitions and typed relations, and sense relations, which are lifted to the synset level as + * documented on {@link WordNetRelation}. Elements outside that subset are skipped, as are + * relations of type {@code other} (the format's untyped escape hatch); any other unknown + * relation type fails loud.

+ * + *

The parser is hardened against XXE: DTD processing and external entities are disabled, so a + * DOCTYPE is skipped but nothing it names is fetched or resolved.

+ * + *

Malformed structure fails loud with an {@link InvalidFormatException} naming the resource + * and, where the parser provides one, the line; I/O failures propagate as {@link IOException}. + * Part-of-speech code {@code s} normalizes to {@link WordNetPOS#ADJECTIVE}, and a {@code similar} + * relation on a verb synset maps to {@link WordNetRelation#VERB_GROUP} rather than + * {@link WordNetRelation#SIMILAR_TO}. The returned lexicon is immutable and safe for concurrent + * lookups.

+ */ +public final class WnLmfReader { + + private static final Map RELATION_NAMES = relationNames(); + + /** The format's escape-hatch relation type; carries no type the contract can express. */ + private static final String OTHER_RELATION = "other"; + + /** The element declaring a lexical entry; opened and closed by the same handlers. */ + private static final String LEXICAL_ENTRY_ELEMENT = "LexicalEntry"; + + /** The element declaring a sense; opened and closed by the same handlers. */ + private static final String SENSE_ELEMENT = "Sense"; + + /** The element declaring a synset; opened and closed by the same handlers. */ + private static final String SYNSET_ELEMENT = "Synset"; + + /** Not instantiable. */ + private WnLmfReader() { + } + + /** + * Reads a WN-LMF XML file. + * + * @param file The XML file. Must not be {@code null} and must exist. + * @return The loaded lexicon. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing. + * @throws InvalidFormatException Thrown if the document is malformed; the message names the + * file and, where available, the line. + * @throws IOException Thrown if reading the file fails. + */ + public static LexicalKnowledgeBase read(Path file) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); + } + try (InputStream in = new BufferedInputStream(Files.newInputStream(file))) { + return read(in, file.toString()); + } + } + + /** + * Reads a WN-LMF XML document from a stream. The stream is not closed. + * + * @param in The document stream. Must not be {@code null}. + * @param resourceName The name used in error messages. Must not be {@code null}. + * @return The loaded lexicon. + * @throws IllegalArgumentException Thrown if an argument is {@code null}. + * @throws InvalidFormatException Thrown if the document is malformed; the message names the + * resource and, where available, the line. + * @throws IOException Thrown if reading the stream fails. + */ + public static LexicalKnowledgeBase read(InputStream in, String resourceName) throws IOException { + if (in == null) { + throw new IllegalArgumentException("In must not be null"); + } + if (resourceName == null) { + throw new IllegalArgumentException("ResourceName must not be null"); + } + final Parser parser = new Parser(resourceName); + try { + final XMLStreamReader reader = hardenedFactory().createXMLStreamReader(in); + try { + parser.parse(reader); + } finally { + reader.close(); + } + } catch (XMLStreamException e) { + // StAX wraps a failing stream read in an XMLStreamException; surface it as the I/O failure. + final Throwable nested = e.getNestedException() == null ? e.getCause() + : e.getNestedException(); + if (nested instanceof IOException io) { + throw io; + } + throw parser.malformed(e.getLocation(), "XML error: " + e.getMessage(), e); + } + return parser.build(); + } + + /** + * Builds an XXE-hardened StAX factory: the DTD internal subset is not processed and external + * entities and the external DTD subset are denied, so a DOCTYPE is skipped but never resolved. + * + * @return The hardened factory. + */ + private static XMLInputFactory hardenedFactory() { + final XMLInputFactory factory = XMLInputFactory.newFactory(); + factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE); + factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE); + factory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + factory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE); + factory.setXMLResolver((publicId, systemId, baseUri, namespace) -> { + throw new XMLStreamException("External entity resolution is disabled, refusing " + systemId); + }); + return factory; + } + + /** Holds the streaming parse state and performs post-parse resolution. */ + private static final class Parser { + + private final String resourceName; + + // Entry state. + private final Set entryIds = new HashSet<>(); + private final Map lemmaByEntryId = new HashMap<>(); + private final Map posByEntryId = new HashMap<>(); + private final Map synsetBySenseId = new HashMap<>(); + private final Map> senseOrder = + new LinkedHashMap<>(); + private final List senseRelations = new ArrayList<>(); + private final Map rawSynsets = new LinkedHashMap<>(); + // Fallback membership (entry ids per synset in document order) when members is absent. + private final Map> entryIdsBySynset = new HashMap<>(); + + // Cursor state. + private String currentEntryId; + private String currentEntryLemma; + private WordNetPOS currentEntryPos; + private String currentSenseId; + private RawSynset currentSynset; + + /** + * Creates a parser. + * + * @param resourceName The name used in error messages. + */ + Parser(String resourceName) { + this.resourceName = resourceName; + } + + /** + * Streams the document, dispatching start and end elements. + * + * @param reader The StAX reader. + * @throws XMLStreamException Thrown if the stream read fails. + * @throws InvalidFormatException Thrown if the document is malformed. + */ + void parse(XMLStreamReader reader) throws XMLStreamException, InvalidFormatException { + while (reader.hasNext()) { + final int event = reader.next(); + // A DTD event carries nothing that can affect parsing once the factory is hardened. + if (event == XMLStreamConstants.START_ELEMENT) { + startElement(reader); + } else if (event == XMLStreamConstants.END_ELEMENT) { + endElement(reader.getLocalName()); + } + } + } + + /** + * Handles one start element, updating cursor state and collecting raw entries, senses, and + * synsets. + * + * @param reader The StAX reader positioned on the start element. + * @throws XMLStreamException Thrown if reading element text fails. + * @throws InvalidFormatException Thrown if the element violates the format. + */ + private void startElement(XMLStreamReader reader) + throws XMLStreamException, InvalidFormatException { + final String name = reader.getLocalName(); + switch (name) { + case LEXICAL_ENTRY_ELEMENT -> { + currentEntryId = requireAttribute(reader, "id"); + if (!entryIds.add(currentEntryId)) { + throw malformed(reader.getLocation(), + "Duplicate lexical entry id " + currentEntryId, null); + } + currentEntryLemma = null; + currentEntryPos = null; + } + case "Lemma" -> { + if (currentEntryId == null) { + throw malformed(reader.getLocation(), "Lemma outside a LexicalEntry", null); + } + currentEntryLemma = requireAttribute(reader, "writtenForm"); + currentEntryPos = parsePos(requireAttribute(reader, "partOfSpeech"), + reader.getLocation()); + lemmaByEntryId.put(currentEntryId, currentEntryLemma); + posByEntryId.put(currentEntryId, currentEntryPos); + } + case SENSE_ELEMENT -> { + if (currentEntryLemma == null) { + throw malformed(reader.getLocation(), + "Sense before its entry's Lemma in LexicalEntry " + currentEntryId, null); + } + currentSenseId = requireAttribute(reader, "id"); + final String synsetId = requireAttribute(reader, "synset"); + if (synsetBySenseId.putIfAbsent(currentSenseId, synsetId) != null) { + throw malformed(reader.getLocation(), "Duplicate sense id " + currentSenseId, null); + } + entryIdsBySynset.computeIfAbsent(synsetId, unused -> new ArrayList<>(2)) + .add(currentEntryId); + final List order = senseOrder.computeIfAbsent( + InMemoryWordNetLexicon.LemmaKey.of(currentEntryLemma, currentEntryPos), + unused -> new ArrayList<>(2)); + if (!order.contains(synsetId)) { + order.add(synsetId); + } + } + case "SenseRelation" -> { + if (currentSenseId == null) { + throw malformed(reader.getLocation(), "SenseRelation outside a Sense", null); + } + senseRelations.add(new RawSenseRelation(currentSenseId, + requireAttribute(reader, "relType"), requireAttribute(reader, "target"), + line(reader.getLocation()))); + } + case SYNSET_ELEMENT -> { + final String id = requireAttribute(reader, "id"); + final WordNetPOS pos = parsePos(requireAttribute(reader, "partOfSpeech"), + reader.getLocation()); + currentSynset = new RawSynset(id, pos, reader.getAttributeValue(null, "members"), + line(reader.getLocation())); + if (rawSynsets.putIfAbsent(id, currentSynset) != null) { + throw malformed(reader.getLocation(), "Duplicate synset id " + id, null); + } + } + case "Definition" -> { + if (currentSynset != null && currentSynset.gloss == null) { + currentSynset.gloss = reader.getElementText(); + } + } + case "SynsetRelation" -> { + if (currentSynset == null) { + throw malformed(reader.getLocation(), "SynsetRelation outside a Synset", null); + } + final String relType = requireAttribute(reader, "relType"); + final String target = requireAttribute(reader, "target"); + // The escape-hatch type is a documented skip, not a rejection. + if (!OTHER_RELATION.equals(relType)) { + currentSynset.relations.add( + new RawRelation(relType, target, line(reader.getLocation()))); + } + } + default -> { + // Pronunciation, Form, Example, SyntacticBehaviour, ILIDefinition, and other + // elements outside the contract subset are skipped. + } + } + } + + /** + * Clears cursor state when a tracked element closes. + * + * @param name The local name of the closing element. + */ + private void endElement(String name) { + switch (name) { + case LEXICAL_ENTRY_ELEMENT -> { + currentEntryId = null; + currentEntryLemma = null; + currentEntryPos = null; + } + case SENSE_ELEMENT -> currentSenseId = null; + case SYNSET_ELEMENT -> currentSynset = null; + default -> { + // Nothing to close for skipped elements. + } + } + } + + /** + * Resolves the collected raw state into an immutable lexicon: validates sense targets, lifts + * sense relations to the synset level, and materializes the contract synsets. + * + * @return The loaded lexicon. + * @throws InvalidFormatException Thrown if a sense or relation references an undeclared + * target, or a synset has no members. + */ + LexicalKnowledgeBase build() throws InvalidFormatException { + // Every sense must point to a declared synset, with a consistent part of speech. + for (final Map.Entry sense : synsetBySenseId.entrySet()) { + final RawSynset target = rawSynsets.get(sense.getValue()); + if (target == null) { + throw malformed(null, + "Sense " + sense.getKey() + " references undeclared synset " + sense.getValue(), + null); + } + } + // Lift sense relations to the synset level. + for (final RawSenseRelation relation : senseRelations) { + if (OTHER_RELATION.equals(relation.relType)) { + continue; + } + final String sourceSynsetId = synsetBySenseId.get(relation.sourceSenseId); + final String targetSynsetId = synsetBySenseId.get(relation.targetSenseId); + if (targetSynsetId == null) { + throw malformed(null, "SenseRelation at line " + relation.line + " from sense " + + relation.sourceSenseId + " references undeclared sense " + relation.targetSenseId, + null); + } + final RawSynset source = rawSynsets.get(sourceSynsetId); + source.relations.add(new RawRelation(relation.relType, targetSynsetId, relation.line)); + } + // Resolve raw synsets into contract synsets. + final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2); + for (final RawSynset raw : rawSynsets.values()) { + final Map> relations = resolveRelations(raw); + synsetsById.put(raw.id, + new Synset(raw.id, raw.pos, memberLemmas(raw), raw.gloss == null ? "" : raw.gloss, + relations)); + } + return new InMemoryWordNetLexicon(synsetsById, senseOrder); + } + + /** + * Resolves a raw synset's relations into typed target-id lists, deduplicated in source order. + * + * @param raw The raw synset. + * @return The typed relations for the contract synset. + * @throws InvalidFormatException Thrown if a relation type is unknown or its target is + * undeclared. + */ + private Map> resolveRelations(RawSynset raw) + throws InvalidFormatException { + final Map> typed = new LinkedHashMap<>(); + for (final RawRelation relation : raw.relations) { + final WordNetRelation type = parseRelation(relation.relType, raw.pos, relation.line); + final RawSynset target = rawSynsets.get(relation.target); + if (target == null) { + throw malformed(null, "Relation " + relation.relType + " at line " + relation.line + + " on synset " + raw.id + " references undeclared synset " + relation.target, null); + } + // Share the synset table's id instance so only one copy of each id is retained. + typed.computeIfAbsent(type, unused -> new LinkedHashSet<>()).add(target.id); + } + final Map> relations = new LinkedHashMap<>(typed.size() * 2); + for (final Map.Entry> entry : typed.entrySet()) { + relations.put(entry.getKey(), List.copyOf(entry.getValue())); + } + return relations; + } + + /** + * Resolves a synset's member entry ids to their lemmas, from the {@code members} attribute + * when present and otherwise from the senses that pointed at the synset. + * + * @param raw The raw synset. + * @return The member lemmas in source order, deduplicated. + * @throws InvalidFormatException Thrown if the synset has no members, names an undeclared + * entry, or a member's part of speech disagrees with the synset's. + */ + private List memberLemmas(RawSynset raw) throws InvalidFormatException { + final List entryIds; + if (raw.members != null && !raw.members.isEmpty()) { + entryIds = LemmaFolding.splitOnSpaces(raw.members); + } else { + final List fromSenses = entryIdsBySynset.get(raw.id); + entryIds = fromSenses == null ? List.of() : fromSenses; + } + if (entryIds.isEmpty()) { + throw malformed(null, "Synset " + raw.id + " at line " + raw.line + + " has no member entries", null); + } + final List lemmas = new ArrayList<>(entryIds.size()); + for (final String entryId : entryIds) { + final String lemma = lemmaByEntryId.get(entryId); + if (lemma == null) { + throw malformed(null, "Synset " + raw.id + " at line " + raw.line + + " lists undeclared member entry " + entryId, null); + } + if (raw.pos != posByEntryId.get(entryId)) { + throw malformed(null, "Synset " + raw.id + " at line " + raw.line + + " has part of speech " + raw.pos + " but member entry " + entryId + + " has " + posByEntryId.get(entryId), null); + } + if (!lemmas.contains(lemma)) { + lemmas.add(lemma); + } + } + return lemmas; + } + + /** + * Maps a WN-LMF part-of-speech code to a {@link WordNetPOS}; code {@code s} normalizes to + * {@link WordNetPOS#ADJECTIVE}. + * + * @param code The part-of-speech code. + * @param location The parser location, for error reporting. + * @return The part of speech. + * @throws InvalidFormatException Thrown if the code is unknown. + */ + private WordNetPOS parsePos(String code, Location location) throws InvalidFormatException { + return switch (code) { + case "n" -> WordNetPOS.NOUN; + case "v" -> WordNetPOS.VERB; + case "a", "s" -> WordNetPOS.ADJECTIVE; + case "r" -> WordNetPOS.ADVERB; + default -> throw malformed(location, "Unknown part-of-speech code: " + code, null); + }; + } + + /** + * Maps a WN-LMF relation name to a {@link WordNetRelation}. A {@code similar} relation on a + * verb synset maps to {@link WordNetRelation#VERB_GROUP}, otherwise to + * {@link WordNetRelation#SIMILAR_TO}. + * + * @param relType The relation name. + * @param sourcePos The part of speech of the source synset. + * @param line The document line, for error reporting. + * @return The mapped relation. + * @throws InvalidFormatException Thrown if the relation name is unknown. + */ + private WordNetRelation parseRelation(String relType, WordNetPOS sourcePos, int line) + throws InvalidFormatException { + if ("similar".equals(relType)) { + return sourcePos == WordNetPOS.VERB ? WordNetRelation.VERB_GROUP + : WordNetRelation.SIMILAR_TO; + } + final WordNetRelation relation = RELATION_NAMES.get(relType); + if (relation == null) { + throw malformed(null, "Unknown relation type " + relType + " at line " + line, null); + } + return relation; + } + + /** + * Reads a required attribute from the current element. + * + * @param reader The StAX reader. + * @param attribute The attribute name. + * @return The non-empty attribute value. + * @throws InvalidFormatException Thrown if the attribute is absent or empty. + */ + private String requireAttribute(XMLStreamReader reader, String attribute) + throws InvalidFormatException { + final String value = reader.getAttributeValue(null, attribute); + if (value == null || value.isEmpty()) { + throw malformed(reader.getLocation(), "Element " + reader.getLocalName() + + " is missing required attribute " + attribute, null); + } + return value; + } + + /** + * Builds a malformed-document exception naming the resource and, when known, the line. + * + * @param location The parser location, or {@code null} when unavailable. + * @param message The failure detail. + * @param cause The underlying cause, or {@code null}. + * @return The exception to throw. + */ + InvalidFormatException malformed(Location location, String message, Throwable cause) { + final int line = line(location); + final String prefix = line < 0 ? "Malformed WN-LMF document " + resourceName + ": " + : "Malformed WN-LMF document " + resourceName + " at line " + line + ": "; + return cause == null ? new InvalidFormatException(prefix + message) + : new InvalidFormatException(prefix + message, cause); + } + + /** + * Extracts a line number from a parser location. + * + * @param location The location, or {@code null}. + * @return The line number, or {@code -1} when unknown. + */ + private static int line(Location location) { + return location == null ? -1 : location.getLineNumber(); + } + } + + private static final class RawSynset { + private final String id; + private final WordNetPOS pos; + private final String members; + private final int line; + private final List relations = new ArrayList<>(4); + private String gloss; + + /** + * Creates a raw synset gathered during parsing. + * + * @param id The synset id. + * @param pos The part of speech. + * @param members The {@code members} attribute value, or {@code null} when absent. + * @param line The document line. + */ + RawSynset(String id, WordNetPOS pos, String members, int line) { + this.id = id; + this.pos = pos; + this.members = members; + this.line = line; + } + } + + /** A parsed synset relation, kept until the target synset is known. */ + private record RawRelation(String relType, String target, int line) { + } + + /** A parsed sense relation, kept until both sense ids are known. */ + private record RawSenseRelation(String sourceSenseId, String relType, String targetSenseId, + int line) { + } + + /** + * Builds the WN-LMF relation-name to {@link WordNetRelation} table. + * + * @return The immutable name table. + */ + private static Map relationNames() { + final Map names = new HashMap<>(); + names.put("antonym", WordNetRelation.ANTONYM); + names.put("hypernym", WordNetRelation.HYPERNYM); + names.put("instance_hypernym", WordNetRelation.INSTANCE_HYPERNYM); + names.put("hyponym", WordNetRelation.HYPONYM); + names.put("instance_hyponym", WordNetRelation.INSTANCE_HYPONYM); + names.put("holo_member", WordNetRelation.MEMBER_HOLONYM); + names.put("holo_substance", WordNetRelation.SUBSTANCE_HOLONYM); + names.put("holo_part", WordNetRelation.PART_HOLONYM); + names.put("mero_member", WordNetRelation.MEMBER_MERONYM); + names.put("mero_substance", WordNetRelation.SUBSTANCE_MERONYM); + names.put("mero_part", WordNetRelation.PART_MERONYM); + names.put("attribute", WordNetRelation.ATTRIBUTE); + names.put("derivation", WordNetRelation.DERIVATIONALLY_RELATED); + names.put("entails", WordNetRelation.ENTAILMENT); + names.put("is_entailed_by", WordNetRelation.ENTAILED_BY); + names.put("causes", WordNetRelation.CAUSE); + names.put("is_caused_by", WordNetRelation.CAUSED_BY); + names.put("also", WordNetRelation.ALSO_SEE); + names.put("participle", WordNetRelation.PARTICIPLE); + names.put("pertainym", WordNetRelation.PERTAINYM); + names.put("domain_topic", WordNetRelation.DOMAIN_TOPIC); + names.put("has_domain_topic", WordNetRelation.MEMBER_OF_DOMAIN_TOPIC); + names.put("domain_region", WordNetRelation.DOMAIN_REGION); + names.put("has_domain_region", WordNetRelation.MEMBER_OF_DOMAIN_REGION); + // The usage domain carries both its current WN-LMF name and the legacy alias. + names.put("exemplifies", WordNetRelation.DOMAIN_USAGE); + names.put("domain_usage", WordNetRelation.DOMAIN_USAGE); + names.put("is_exemplified_by", WordNetRelation.MEMBER_OF_DOMAIN_USAGE); + names.put("has_domain_usage", WordNetRelation.MEMBER_OF_DOMAIN_USAGE); + return Map.copyOf(names); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java new file mode 100644 index 0000000000..ed6b18ea8b --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java @@ -0,0 +1,627 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; + +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Reads a Princeton WordNet database directory in the + * WNDB format + * ({@code index.noun}, {@code data.noun}, and the corresponding pairs for verbs, adjectives, and + * adverbs) into a {@link LexicalKnowledgeBase}. + * + *

All eight index and data files must be present. License preamble lines (which begin with a + * space in the released files) are skipped. {@code index.sense} is not read, and the + * {@code *.exc} exception lists are the {@link MorphyLemmatizer} companion input, read + * separately.

+ * + *

Synset ids are minted as {@code wndb-}offset{@code -}pos from the data file's + * 8-digit byte offset and part-of-speech letter, for example {@code wndb-00001740-n}; the id is + * opaque to consumers. Adjective satellite lines normalize to {@link WordNetPOS#ADJECTIVE}, the + * syntactic markers the adjective files append ({@code (p)}, {@code (a)}, {@code (ip)}) are + * stripped, and underscores in lemmas become spaces. Sense order per lemma follows the index + * file's offset order.

+ * + *

Malformed content fails loud with an {@link InvalidFormatException} naming the file and + * line; I/O failures propagate as {@link IOException}. The returned lexicon is immutable and safe + * for concurrent lookups.

+ */ +public final class WndbReader { + + private static final Map POINTER_SYMBOLS = pointerSymbols(); + + /** The prefix of every synset id this reader mints. */ + private static final String SYNSET_ID_PREFIX = "wndb-"; + + /** Not instantiable. */ + private WndbReader() { + } + + /** + * Mints a synset id in this reader's scheme: the {@code wndb-} prefix, the 8-digit data-file + * byte offset, a hyphen, and the part-of-speech letter, for example {@code wndb-00001740-n}. + * + * @param offset The 8-digit synset offset field. + * @param posChar The WNDB part-of-speech letter. + * @return The minted synset id. + */ + private static String synsetId(String offset, char posChar) { + return SYNSET_ID_PREFIX + offset + '-' + posChar; + } + + /** + * Reads a WNDB database directory. + * + * @param directory The directory containing the eight index and data files. Must not be + * {@code null} and must exist. + * @return The loaded lexicon. + * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a + * directory. + * @throws InvalidFormatException Thrown if a database file is missing or any file is + * malformed; the message names the file and line. + * @throws IOException Thrown if reading a file fails. + */ + public static LexicalKnowledgeBase read(Path directory) throws IOException { + if (directory == null) { + throw new IllegalArgumentException("Directory must not be null"); + } + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException( + "Directory does not exist or is not a directory: " + directory); + } + final Map rawSynsets = new LinkedHashMap<>(); + for (final FilePos filePos : FilePos.values()) { + parseDataFile(directory, filePos, rawSynsets); + } + final Map synsetsById = resolve(rawSynsets); + final Map> senseOrder = new LinkedHashMap<>(); + for (final FilePos filePos : FilePos.values()) { + parseIndexFile(directory, filePos, rawSynsets, senseOrder); + } + return new InMemoryWordNetLexicon(synsetsById, senseOrder); + } + + /** The four part-of-speech file pairs of a WNDB directory. */ + private enum FilePos { + NOUN("noun", 'n', WordNetPOS.NOUN), + VERB("verb", 'v', WordNetPOS.VERB), + ADJECTIVE("adj", 'a', WordNetPOS.ADJECTIVE), + ADVERB("adv", 'r', WordNetPOS.ADVERB); + + private final String suffix; + private final char posChar; + private final WordNetPOS pos; + + /** + * Binds a part of speech to its file suffix and WNDB letter. + * + * @param suffix The file suffix, for example {@code noun}. + * @param posChar The WNDB part-of-speech letter. + * @param pos The mapped part of speech. + */ + FilePos(String suffix, char posChar, WordNetPOS pos) { + this.suffix = suffix; + this.posChar = posChar; + this.pos = pos; + } + } + + /** + * Parses one {@code data.*} file, collecting its synsets keyed by minted id. + * + * @param directory The database directory. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The accumulating synset table. + * @throws IOException Thrown if the file is missing, malformed, or unreadable. + */ + private static void parseDataFile(Path directory, FilePos filePos, + Map rawSynsets) throws IOException { + final String fileName = "data." + filePos.suffix; + final byte[] bytes = readAll(directory.resolve(fileName), fileName); + int lineStart = 0; + int lineNumber = 0; + while (lineStart < bytes.length) { + lineNumber++; + int lineEnd = lineStart; + while (lineEnd < bytes.length && bytes[lineEnd] != '\n') { + lineEnd++; + } + // ISO-8859-1 decodes bytes one-to-one, keeping offsets exact for any released file. + final String line = + new String(bytes, lineStart, lineEnd - lineStart, StandardCharsets.ISO_8859_1); + if (!line.isEmpty() && line.charAt(0) != ' ') { + parseDataLine(line, lineStart, fileName, lineNumber, filePos, rawSynsets); + } + lineStart = lineEnd + 1; + } + } + + /** + * Parses one data-file synset line into a raw synset. + * + * @param line The decoded line, without its trailing newline. + * @param byteOffset The line's byte offset, matched against the line's own offset field. + * @param fileName The data file name, for error reporting. + * @param lineNumber The 1-based line number. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The accumulating synset table. + * @throws InvalidFormatException Thrown if the line is malformed or its offset field disagrees + * with its byte position. + */ + private static void parseDataLine(String line, int byteOffset, String fileName, int lineNumber, + FilePos filePos, Map rawSynsets) + throws InvalidFormatException { + final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber); + final String offsetField = tokens.next("synset_offset"); + if (parseOffset(offsetField, tokens) != byteOffset) { + throw malformed(fileName, lineNumber, "Synset offset field " + offsetField + + " disagrees with the actual byte position " + byteOffset); + } + tokens.next("lex_filenum (lexicographer file number)"); + final String ssType = tokens.next("ss_type (synset type)"); + final boolean validType = switch (filePos) { + case ADJECTIVE -> "a".equals(ssType) || "s".equals(ssType); + default -> ssType.length() == 1 && ssType.charAt(0) == filePos.posChar; + }; + if (!validType) { + throw malformed(fileName, lineNumber, + "Synset type " + ssType + " does not belong in " + fileName); + } + final int wordCount = tokens.nextInt("w_cnt (word count)", 16); + if (wordCount < 1) { + throw malformed(fileName, lineNumber, "Word count must be at least 1, got: " + wordCount); + } + final List lemmas = new ArrayList<>(wordCount); + for (int i = 0; i < wordCount; i++) { + final String lemma = cleanLemma(tokens.next("word"), fileName, lineNumber); + tokens.nextInt("lex_id (sense id within the lexicographer file)", 16); + if (!lemmas.contains(lemma)) { + lemmas.add(lemma); + } + } + final int pointerCount = tokens.nextInt("p_cnt (pointer count)", 10); + final List pointers = new ArrayList<>(pointerCount); + for (int i = 0; i < pointerCount; i++) { + final String symbol = tokens.next("pointer_symbol"); + final WordNetRelation relation = POINTER_SYMBOLS.get(symbol); + if (relation == null) { + throw malformed(fileName, lineNumber, "Undeclared pointer symbol: " + symbol); + } + final String targetOffset = tokens.next("pointer synset_offset"); + parseOffset(targetOffset, tokens); + final char targetPos = posChar(tokens.next("pointer pos"), tokens); + tokens.next("pointer source/target"); + pointers.add(new RawPointer(relation, synsetId(targetOffset, targetPos), lineNumber)); + } + if (filePos == FilePos.VERB) { + final int frameCount = tokens.nextInt("f_cnt (verb frame count)", 10); + for (int i = 0; i < frameCount; i++) { + tokens.next("frame marker"); + tokens.next("f_num (verb frame number)"); + tokens.next("w_num (word number)"); + } + } + final String gloss = tokens.gloss(); + final String id = synsetId(offsetField, filePos.posChar); + rawSynsets.put(id, new RawSynset(id, filePos.pos, lemmas, gloss, pointers, + fileName, lineNumber)); + } + + /** + * Resolves raw synsets into contract synsets, validating every pointer target. + * + * @param rawSynsets The parsed synsets keyed by id. + * @return The contract synsets keyed by id. + * @throws InvalidFormatException Thrown if a pointer targets a nonexistent synset. + */ + private static Map resolve(Map rawSynsets) + throws InvalidFormatException { + final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2); + for (final RawSynset raw : rawSynsets.values()) { + final Map> typed = new LinkedHashMap<>(); + for (final RawPointer pointer : raw.pointers) { + final RawSynset target = rawSynsets.get(pointer.targetId); + if (target == null) { + throw malformed(raw.fileName, pointer.lineNumber, "Synset " + raw.id + " has a " + + pointer.relation + " pointer to nonexistent synset " + pointer.targetId); + } + // Share the synset table's id instance so only one copy of each id is retained. + typed.computeIfAbsent(pointer.relation, unused -> new LinkedHashSet<>()) + .add(target.id); + } + final Map> relations = new LinkedHashMap<>(typed.size() * 2); + for (final Map.Entry> entry : typed.entrySet()) { + relations.put(entry.getKey(), List.copyOf(entry.getValue())); + } + synsetsById.put(raw.id, new Synset(raw.id, raw.pos, raw.lemmas, raw.gloss, relations)); + } + return synsetsById; + } + + /** + * Parses one {@code index.*} file, building the sense order per folded lemma key. + * + * @param directory The database directory. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The resolved synset table, for offset validation. + * @param senses The accumulating sense-order map. + * @throws IOException Thrown if the file is missing, malformed, or unreadable. + */ + private static void parseIndexFile(Path directory, FilePos filePos, + Map rawSynsets, + Map> senses) + throws IOException { + final String fileName = "index." + filePos.suffix; + final byte[] bytes = readAll(directory.resolve(fileName), fileName); + final String content = new String(bytes, StandardCharsets.ISO_8859_1); + int lineNumber = 0; + int lineStart = 0; + while (lineStart < content.length()) { + lineNumber++; + int lineEnd = content.indexOf('\n', lineStart); + if (lineEnd < 0) { + lineEnd = content.length(); + } + final String line = content.substring(lineStart, lineEnd); + if (!line.isEmpty() && line.charAt(0) != ' ') { + parseIndexLine(line, fileName, lineNumber, filePos, rawSynsets, senses); + } + lineStart = lineEnd + 1; + } + } + + /** + * Parses one index-file line into a lemma's sense order. + * + * @param line The line to parse. + * @param fileName The index file name, for error reporting. + * @param lineNumber The 1-based line number. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The resolved synset table, for offset validation. + * @param senses The accumulating sense-order map. + * @throws InvalidFormatException Thrown if the line is malformed or references an unknown + * offset. + */ + private static void parseIndexLine(String line, String fileName, int lineNumber, + FilePos filePos, Map rawSynsets, + Map> senses) + throws InvalidFormatException { + final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber); + final String lemma = tokens.next("lemma"); + final String pos = tokens.next("pos"); + if (pos.length() != 1 || pos.charAt(0) != filePos.posChar) { + throw malformed(fileName, lineNumber, "Index pos " + pos + " does not belong in " + + fileName); + } + final int synsetCount = tokens.nextInt("synset_cnt (synset count)", 10); + if (synsetCount < 1) { + throw malformed(fileName, lineNumber, + "Synset count must be at least 1, got: " + synsetCount); + } + final int pointerTypeCount = tokens.nextInt("p_cnt (pointer count)", 10); + for (int i = 0; i < pointerTypeCount; i++) { + // The summary symbols are informational; the data file's pointers are authoritative. + tokens.next("ptr_symbol (pointer symbol)"); + } + tokens.next("sense_cnt (sense count)"); + tokens.next("tagsense_cnt (tagged-sense count)"); + final List order = new ArrayList<>(synsetCount); + for (int i = 0; i < synsetCount; i++) { + final String offset = tokens.next("synset_offset"); + parseOffset(offset, tokens); + final String synsetId = synsetId(offset, filePos.posChar); + if (!rawSynsets.containsKey(synsetId)) { + throw malformed(fileName, lineNumber, "Lemma " + lemma + " references offset " + offset + + " with no data." + filePos.suffix + " line"); + } + if (!order.contains(synsetId)) { + order.add(synsetId); + } + } + final InMemoryWordNetLexicon.LemmaKey key = + InMemoryWordNetLexicon.LemmaKey.of(lemma, filePos.pos); + final List existing = senses.get(key); + if (existing == null) { + senses.put(key, order); + } else { + // Two index lemmas can fold to one key; keep first-listed order and append the rest. + for (final String synsetId : order) { + if (!existing.contains(synsetId)) { + existing.add(synsetId); + } + } + } + } + + /** + * Strips the adjective syntactic markers ({@code (p)}, {@code (a)}, {@code (ip)}) and turns + * underscores into spaces. + * + * @param word The raw word field. + * @param fileName The data file name, for error reporting. + * @param lineNumber The 1-based line number. + * @return The cleaned lemma. + * @throws InvalidFormatException Thrown if the word carries an unknown marker or is empty. + */ + private static String cleanLemma(String word, String fileName, int lineNumber) + throws InvalidFormatException { + String cleaned = word; + if (cleaned.endsWith(")")) { + final int open = cleaned.lastIndexOf('('); + final String marker = open < 0 ? "" : cleaned.substring(open); + if (!"(p)".equals(marker) && !"(a)".equals(marker) && !"(ip)".equals(marker)) { + throw malformed(fileName, lineNumber, "Unknown syntactic marker on word: " + word); + } + cleaned = cleaned.substring(0, open); + } + if (cleaned.isEmpty()) { + throw malformed(fileName, lineNumber, "Empty word field"); + } + return cleaned.replace('_', ' '); + } + + /** + * Parses an 8-digit synset offset. + * + * @param offset The offset field. + * @param tokens The tokenizer, for error reporting. + * @return The offset as an integer. + * @throws InvalidFormatException Thrown if the field is not 8 digits. + */ + private static int parseOffset(String offset, Tokenizer tokens) throws InvalidFormatException { + if (offset.length() != 8) { + throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); + } + int value = 0; + for (int i = 0; i < 8; i++) { + final char c = offset.charAt(i); + if (c < '0' || c > '9') { + throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); + } + value = value * 10 + (c - '0'); + } + return value; + } + + /** + * Parses a pointer's one-letter part-of-speech code. + * + * @param pos The code field. + * @param tokens The tokenizer, for error reporting. + * @return One of {@code n}, {@code v}, {@code a}, {@code r}. + * @throws InvalidFormatException Thrown if the code is not one of those letters. + */ + private static char posChar(String pos, Tokenizer tokens) throws InvalidFormatException { + if (pos.length() == 1) { + final char c = pos.charAt(0); + if (c == 'n' || c == 'v' || c == 'a' || c == 'r') { + return c; + } + } + throw tokens.malformedToken("Pointer pos must be one of n, v, a, r, got: " + pos); + } + + /** + * Reads a required database file in full. + * + * @param file The file path. + * @param fileName The file name, for error reporting. + * @return The file bytes. + * @throws InvalidFormatException Thrown if the file is missing. + * @throws IOException Thrown if reading fails. + */ + private static byte[] readAll(Path file, String fileName) throws IOException { + if (!Files.isRegularFile(file)) { + throw new InvalidFormatException("Missing WNDB database file: " + file); + } + return Files.readAllBytes(file); + } + + /** + * Builds a malformed-file exception naming the file and line. + * + * @param fileName The file name. + * @param lineNumber The 1-based line number. + * @param message The failure detail. + * @return The exception to throw. + */ + private static InvalidFormatException malformed(String fileName, int lineNumber, + String message) { + return new InvalidFormatException( + "Malformed WNDB file " + fileName + " at line " + lineNumber + ": " + message); + } + + /** A cursor over one line's space-separated fields. */ + private static final class Tokenizer { + + private final String line; + private final String fileName; + private final int lineNumber; + private int position; + + /** + * Creates a tokenizer over one line. + * + * @param line The line to tokenize. + * @param fileName The file name, for error reporting. + * @param lineNumber The 1-based line number. + */ + Tokenizer(String line, String fileName, int lineNumber) { + this.line = line; + this.fileName = fileName; + this.lineNumber = lineNumber; + } + + /** + * Reads the next space-separated field. + * + * @param field The field name, for error reporting. + * @return The field value. + * @throws InvalidFormatException Thrown if the line is truncated before the field. + */ + String next(String field) throws InvalidFormatException { + while (position < line.length() && line.charAt(position) == ' ') { + position++; + } + if (position >= line.length()) { + throw malformed(fileName, lineNumber, "Truncated line, missing field: " + field); + } + final int start = position; + while (position < line.length() && line.charAt(position) != ' ') { + position++; + } + return line.substring(start, position); + } + + /** + * Reads the next field as an integer in the given radix. + * + * @param field The field name, for error reporting. + * @param radix The numeric radix. + * @return The parsed value. + * @throws InvalidFormatException Thrown if the field is missing or not a valid integer. + */ + int nextInt(String field, int radix) throws InvalidFormatException { + final String token = next(field); + try { + return Integer.parseInt(token, radix); + } catch (NumberFormatException e) { + throw new InvalidFormatException(malformed(fileName, lineNumber, + "Field " + field + " is not a base-" + radix + " integer: " + token).getMessage(), e); + } + } + + /** + * Reads the gloss: the remainder after the pipe separator, trimmed of surrounding spaces. + * + * @return The gloss text. + * @throws InvalidFormatException Thrown if the pipe separator is missing. + */ + String gloss() throws InvalidFormatException { + final String separator = next("gloss separator"); + if (!"|".equals(separator)) { + throw malformed(fileName, lineNumber, "Expected the | gloss separator, got: " + separator); + } + int start = position; + while (start < line.length() && line.charAt(start) == ' ') { + start++; + } + int end = line.length(); + while (end > start && line.charAt(end - 1) == ' ') { + end--; + } + return line.substring(start, end); + } + + /** + * Builds a malformed-file exception at this tokenizer's line. + * + * @param message The failure detail. + * @return The exception to throw. + */ + InvalidFormatException malformedToken(String message) { + return malformed(fileName, lineNumber, message); + } + } + + /** A parsed pointer line, kept until the target synset is known. */ + private record RawPointer(WordNetRelation relation, String targetId, int lineNumber) { + } + + private static final class RawSynset { + private final String id; + private final WordNetPOS pos; + private final List lemmas; + private final String gloss; + private final List pointers; + private final String fileName; + private final int lineNumber; + + /** + * Creates a raw synset gathered while parsing a data file. + * + * @param id The minted synset id. + * @param pos The part of speech. + * @param lemmas The member lemmas. + * @param gloss The gloss text. + * @param pointers The raw pointers to resolve. + * @param fileName The source file name. + * @param lineNumber The source line number. + */ + RawSynset(String id, WordNetPOS pos, List lemmas, String gloss, + List pointers, String fileName, int lineNumber) { + this.id = id; + this.pos = pos; + this.lemmas = lemmas; + this.gloss = gloss; + this.pointers = pointers; + this.fileName = fileName; + this.lineNumber = lineNumber; + } + } + + /** + * Builds the WNDB pointer-symbol to {@link WordNetRelation} table. + * + * @return The immutable symbol table. + */ + private static Map pointerSymbols() { + final Map symbols = new HashMap<>(); + symbols.put("!", WordNetRelation.ANTONYM); + symbols.put("@", WordNetRelation.HYPERNYM); + symbols.put("@i", WordNetRelation.INSTANCE_HYPERNYM); + symbols.put("~", WordNetRelation.HYPONYM); + symbols.put("~i", WordNetRelation.INSTANCE_HYPONYM); + symbols.put("#m", WordNetRelation.MEMBER_HOLONYM); + symbols.put("#s", WordNetRelation.SUBSTANCE_HOLONYM); + symbols.put("#p", WordNetRelation.PART_HOLONYM); + symbols.put("%m", WordNetRelation.MEMBER_MERONYM); + symbols.put("%s", WordNetRelation.SUBSTANCE_MERONYM); + symbols.put("%p", WordNetRelation.PART_MERONYM); + symbols.put("=", WordNetRelation.ATTRIBUTE); + symbols.put("+", WordNetRelation.DERIVATIONALLY_RELATED); + symbols.put("*", WordNetRelation.ENTAILMENT); + symbols.put(">", WordNetRelation.CAUSE); + symbols.put("^", WordNetRelation.ALSO_SEE); + symbols.put("$", WordNetRelation.VERB_GROUP); + symbols.put("&", WordNetRelation.SIMILAR_TO); + symbols.put("<", WordNetRelation.PARTICIPLE); + symbols.put("\\", WordNetRelation.PERTAINYM); + symbols.put(";c", WordNetRelation.DOMAIN_TOPIC); + symbols.put("-c", WordNetRelation.MEMBER_OF_DOMAIN_TOPIC); + symbols.put(";r", WordNetRelation.DOMAIN_REGION); + symbols.put("-r", WordNetRelation.MEMBER_OF_DOMAIN_REGION); + symbols.put(";u", WordNetRelation.DOMAIN_USAGE); + symbols.put("-u", WordNetRelation.MEMBER_OF_DOMAIN_USAGE); + return Map.copyOf(symbols); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java new file mode 100644 index 0000000000..8a238832fb --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the constructor's referential-integrity validation directly, with deliberately + * inconsistent maps a reader would never produce: any future reader relies on these checks, + * so they are pinned independently of both existing readers. + */ +public class InMemoryWordNetLexiconTest { + + private static Synset synset(String id, Map> relations) { + return new Synset(id, WordNetPOS.NOUN, List.of("lemma"), "a gloss", relations); + } + + @Test + void testAcceptsConsistentMaps() { + final Synset a = synset("a", Map.of(WordNetRelation.HYPERNYM, List.of("b"))); + final Synset b = synset("b", Map.of()); + final InMemoryWordNetLexicon lexicon = new InMemoryWordNetLexicon( + Map.of("a", a, "b", b), + Map.of(InMemoryWordNetLexicon.LemmaKey.of("lemma", WordNetPOS.NOUN), List.of("a", "b"))); + assertEquals(2, lexicon.size()); + assertEquals(List.of(a, b), lexicon.lookup("lemma", WordNetPOS.NOUN)); + } + + @Test + void testRejectsKeyThatDoesNotMatchSynsetId() { + final Map table = Map.of("wrong-key", synset("real-id", Map.of())); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(table, Map.of())); + assertTrue(e.getMessage().contains("wrong-key")); + } + + @Test + void testRejectsDanglingRelationTarget() { + final Map table = + Map.of("a", synset("a", Map.of(WordNetRelation.HYPERNYM, List.of("nope")))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(table, Map.of())); + assertTrue(e.getMessage().contains("nope")); + assertTrue(e.getMessage().contains("HYPERNYM")); + } + + @Test + void testRejectsSenseOrderEntryWithUnknownSynset() { + final Map table = Map.of("a", synset("a", Map.of())); + final Map> senseOrder = + Map.of(InMemoryWordNetLexicon.LemmaKey.of("lemma", WordNetPOS.NOUN), List.of("missing")); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(table, senseOrder)); + assertTrue(e.getMessage().contains("missing")); + assertTrue(e.getMessage().contains("lemma")); + } + + @Test + void testRejectsNullMaps() { + assertThrows(IllegalArgumentException.class, () -> new InMemoryWordNetLexicon(null, Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(Map.of(), null)); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java new file mode 100644 index 0000000000..3d32eb4431 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.WordNetPOS; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Pins the shared fold and split behavior every user of {@link LemmaFolding} depends on: + * the exception lists, the sense index keys, and the WN-LMF members parsing all fold and + * split through this one implementation. + */ +public class LemmaFoldingTest { + + @Test + void testFoldLowercasesWithRootLocaleAndTreatsUnderscoreAsSpace() { + assertEquals("mice", LemmaFolding.fold("MICE")); + assertEquals("domestic dog", LemmaFolding.fold("Domestic_Dog")); + assertEquals("attorney general", LemmaFolding.fold("attorney_general")); + assertEquals("dog", LemmaFolding.fold("dog")); + assertEquals("", LemmaFolding.fold("")); + } + + @Test + void testSplitOnSpacesCollapsesRunsAndIgnoresEdges() { + assertEquals(List.of("a", "b", "c"), LemmaFolding.splitOnSpaces("a b c")); + assertEquals(List.of("a", "b"), LemmaFolding.splitOnSpaces("a b")); + assertEquals(List.of("a"), LemmaFolding.splitOnSpaces("a")); + assertEquals(List.of("a"), LemmaFolding.splitOnSpaces(" a ")); + assertEquals(List.of(), LemmaFolding.splitOnSpaces("")); + assertEquals(List.of(), LemmaFolding.splitOnSpaces(" ")); + } + + @Test + void testLemmaKeyAndExceptionLookupAgreeOnTheFold() { + // The agreement that makes Morphy correct: a key built from a stored written form and a + // query folded at lookup time land on the same canonical shape. + assertEquals(InMemoryWordNetLexicon.LemmaKey.of("Domestic_Dog", WordNetPOS.NOUN), + InMemoryWordNetLexicon.LemmaKey.of(LemmaFolding.fold("DOMESTIC_DOG"), WordNetPOS.NOUN)); + } + + @Test + void testFoldRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> LemmaFolding.fold(null)); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java new file mode 100644 index 0000000000..0292f7ffe2 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the immutable-after-load contract: one loaded lexicon serves many threads issuing + * concurrent lookups, and every thread observes exactly the single-threaded results. + */ +public class LexiconConcurrencyTest { + + private static final int THREADS = 8; + private static final int ITERATIONS = 500; + + @Test + void testConcurrentLookupsSeeConsistentResults() throws InterruptedException { + final LexicalKnowledgeBase lexicon = WndbReaderTest.fixture(); + final CountDownLatch start = new CountDownLatch(1); + final CountDownLatch done = new CountDownLatch(THREADS); + final Queue problems = new ConcurrentLinkedQueue<>(); + for (int t = 0; t < THREADS; t++) { + final Thread thread = new Thread(() -> { + try { + start.await(); + for (int i = 0; i < ITERATIONS; i++) { + verifyOnce(lexicon, problems); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + problems.add("Interrupted: " + e); + } catch (RuntimeException e) { + problems.add("Unexpected exception: " + e); + } finally { + done.countDown(); + } + }); + thread.setDaemon(true); + thread.start(); + } + start.countDown(); + assertTrue(done.await(60, TimeUnit.SECONDS), "Worker threads must finish in time"); + assertEquals(List.of(), List.copyOf(problems)); + } + + private static void verifyOnce(LexicalKnowledgeBase lexicon, Queue problems) { + if (!"wndb-00001075-n".equals(lexicon.lookup("dog", WordNetPOS.NOUN).get(0).id())) { + problems.add("Wrong dog lookup"); + } + if (lexicon.lookup("run", WordNetPOS.NOUN).size() != 2) { + problems.add("Wrong run sense count"); + } + if (!List.of("wndb-00001160-n") + .equals(lexicon.related("wndb-00001075-n", WordNetRelation.HYPERNYM))) { + problems.add("Wrong dog hypernym"); + } + if (lexicon.contains("zebra", WordNetPOS.NOUN)) { + problems.add("Phantom zebra"); + } + if (!lexicon.contains("walk", WordNetPOS.VERB)) { + problems.add("Missing walk verb"); + } + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java new file mode 100644 index 0000000000..b3aaf78ba2 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.WordNetPOS; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MorphyExceptionsTest { + + static MorphyExceptions fixture() { + try { + return MorphyExceptions.load(WndbReaderTest.fixtureDirectory()); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException reading the fixture lists", e); + } + } + + /** + * Writes the standard one-entry exception list for each part of speech into + * {@code directory}. Tests that need a variation overwrite or delete individual files + * afterwards. + * + * @param directory The directory to receive {@code noun.exc}, {@code verb.exc}, + * {@code adj.exc}, and {@code adv.exc}. + * @throws IOException Thrown if writing a file fails. + */ + private static void writeStandardLists(Path directory) throws IOException { + Files.writeString(directory.resolve("noun.exc"), "mice mouse\n"); + Files.writeString(directory.resolve("verb.exc"), "went go\n"); + Files.writeString(directory.resolve("adj.exc"), "better good\n"); + Files.writeString(directory.resolve("adv.exc"), "best well\n"); + } + + @ParameterizedTest + @CsvSource(nullValues = "unknown", value = { + "mice, NOUN, mouse", + "went, VERB, go", + "better, ADJECTIVE, good", + "best, ADVERB, well", + // Entries are part-of-speech scoped: went is only a verb exception. + "went, NOUN, unknown", + "dog, NOUN, unknown", + }) + void testLookupPerPartOfSpeech(String form, WordNetPOS pos, String lemma) { + final List expected = lemma == null ? List.of() : List.of(lemma); + assertEquals(expected, fixture().lookup(form, pos)); + } + + @Test + void testLookupFoldsCase() { + assertEquals(List.of("mouse"), fixture().lookup("Mice", WordNetPOS.NOUN)); + assertEquals(List.of("mouse"), fixture().lookup("MICE", WordNetPOS.NOUN)); + } + + @Test + void testLookupRejectsNulls() { + final MorphyExceptions exceptions = fixture(); + assertThrows(IllegalArgumentException.class, + () -> exceptions.lookup(null, WordNetPOS.NOUN)); + assertThrows(IllegalArgumentException.class, () -> exceptions.lookup("mice", null)); + } + + @Test + void testLoadRejectsNullAndMissingDirectory(@TempDir Path tempDir) { + assertThrows(IllegalArgumentException.class, () -> MorphyExceptions.load(null)); + assertThrows(IllegalArgumentException.class, + () -> MorphyExceptions.load(tempDir.resolve("absent"))); + } + + @Test + void testLoadRejectsMissingFile(@TempDir Path tempDir) throws IOException { + writeStandardLists(tempDir); + Files.delete(tempDir.resolve("adv.exc")); + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> MorphyExceptions.load(tempDir)); + assertTrue(e.getMessage().contains("adv.exc")); + } + + @Test + void testLoadRejectsMalformedLine(@TempDir Path tempDir) throws IOException { + writeStandardLists(tempDir); + Files.writeString(tempDir.resolve("noun.exc"), "mice mouse\nlonely\n"); + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> MorphyExceptions.load(tempDir)); + assertTrue(e.getMessage().contains("noun.exc")); + assertTrue(e.getMessage().contains("line 2")); + } + + @Test + void testMultipleBaseFormsKeepFileOrder(@TempDir Path tempDir) throws IOException { + writeStandardLists(tempDir); + Files.writeString(tempDir.resolve("noun.exc"), "axes axis ax\n"); + assertEquals(List.of("axis", "ax"), + MorphyExceptions.load(tempDir).lookup("axes", WordNetPOS.NOUN)); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java new file mode 100644 index 0000000000..24df815548 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import opennlp.tools.wordnet.WordNetPOS; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class MorphyLemmatizerTest { + + private static MorphyLemmatizer morphy() { + return new MorphyLemmatizer(WndbReaderTest.fixture(), MorphyExceptionsTest.fixture()); + } + + private static String one(String token, String tag) { + return morphy().lemmatize(new String[] {token}, new String[] {tag})[0]; + } + + @ParameterizedTest + @CsvSource({ + // Irregular forms resolve through the exception lists. + "mice, NN, mouse", + "Mice, NNS, mouse", + "men, NNS, man", + "ran, VBD, run", + "running, VBG, run", + "went, VBD, go", + "gone, VBN, go", + "best, RBS, well", + // Regular detachments, validated against the lexicon. + "dogs, NNS, dog", + "boxes, NNS, box", + "berries, NNS, berry", + "runs, NNS, run", + "runs, VBZ, run", + "walked, VBD, walk", + "walking, VBG, walk", + "walks, VBZ, walk", + "moved, VBD, move", + "taller, JJR, tall", + "tallest, JJS, tall", + "larger, JJR, large", + // A word that is already a lemma comes back as itself. + "dog, NN, dog", + "quickly, RB, quickly", + // WordNet letter tags are accepted alongside Penn tags. + "dogs, n, dog", + "walked, v, walk", + "taller, a, tall", + "best, r, well", + }) + void testLemmatizesToken(String token, String tag, String lemma) { + assertEquals(lemma, one(token, tag)); + } + + @ParameterizedTest + @CsvSource({ + // Rule candidates not in the lexicon are rejected, not returned. + "dogged, VBD", + "boxes, VBZ", + "glarbs, NNS", + // A known word under the wrong part of speech is unknown. + "walk, NN", + // Tags outside the mapping yield the unknown marker. + "dog, DT", + "dog, XYZ", + "dogs, ''", + // Multi-letter closed-class tags that merely begin with a WordNet letter code are not + // adjective lookups: AUX was must be unknown, and AUX taller must not detach to tall. + "was, AUX", + "taller, AUX", + }) + void testUnknownYieldsMarker(String token, String tag) { + assertEquals("O", one(token, tag)); + } + + @Test + void testExceptionHitsAreReturnedWithoutLexiconValidation() { + // oxen maps to ox, which the miniature lexicon does not contain; the exception list is + // authoritative for irregulars, so the lemma is returned anyway. + assertEquals("ox", one("oxen", "NNS")); + // better maps to good, also absent from the miniature lexicon. + assertEquals("good", one("better", "JJR")); + } + + @Test + void testArrayFormKeepsPositions() { + final String[] lemmas = morphy().lemmatize( + new String[] {"The", "mice", "ran", "quickly"}, + new String[] {"DT", "NNS", "VBD", "RB"}); + assertArrayEquals(new String[] {"O", "mouse", "run", "quickly"}, lemmas); + } + + @Test + void testListFormReturnsAllCandidates() { + final List> lemmas = morphy().lemmatize( + List.of("glarbs", "berries"), List.of("NNS", "NNS")); + assertEquals(List.of("O"), lemmas.get(0)); + assertEquals(List.of("berry"), lemmas.get(1)); + } + + @Test + void testWorksIdenticallyOverTheWnLmfLexicon() { + final MorphyLemmatizer lmfMorphy = + new MorphyLemmatizer(WnLmfReaderTest.fixture(), MorphyExceptionsTest.fixture()); + assertArrayEquals(new String[] {"mouse", "box", "walk", "large", "O"}, + lmfMorphy.lemmatize( + new String[] {"mice", "boxes", "walking", "larger", "dogged"}, + new String[] {"NNS", "NNS", "VBG", "JJR", "VBD"})); + } + + @ParameterizedTest + @CsvSource(nullValues = "none", value = { + "NNP, NOUN", + "VBZ, VERB", + "JJ, ADJECTIVE", + "RBR, ADVERB", + "a, ADJECTIVE", + "s, ADJECTIVE", + "ADJ, ADJECTIVE", + "ADV, ADVERB", + "r, ADVERB", + "DT, none", + "'', none", + // The letter codes a and s match only as one-letter tags: multi-letter tags beginning + // with those letters are closed-class or symbol tags, never adjectives. + "AUX, none", + "ADP, none", + "SCONJ, none", + "SYM, none", + }) + void testPosFromTagMapping(String tag, WordNetPOS pos) { + assertEquals(pos, MorphyLemmatizer.posFromTag(tag)); + } + + @Test + void testPosFromTagRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> MorphyLemmatizer.posFromTag(null)); + } + + @Test + void testConstructorFailsLoudWithoutInputs() { + final MorphyExceptions exceptions = MorphyExceptionsTest.fixture(); + assertThrows(IllegalArgumentException.class, + () -> new MorphyLemmatizer(null, exceptions)); + assertThrows(IllegalArgumentException.class, + () -> new MorphyLemmatizer(WndbReaderTest.fixture(), null)); + } + + @Test + void testRejectsNullOrMismatchedSequences() { + final MorphyLemmatizer morphy = morphy(); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize((String[]) null, new String[0])); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[0], (String[]) null)); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[] {"a", "b"}, new String[] {"NN"})); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(List.of("a"), List.of("NN", "NN"))); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[] {null}, new String[] {"NN"})); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[] {"dog"}, new String[] {null})); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java new file mode 100644 index 0000000000..b9f3523cbc --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Asserts that the WN-LMF fixture and the WNDB fixture, which encode the same miniature + * wordnet, load into equivalent lexicon views. Synset ids are reader-minted and intentionally + * differ, so the comparison is structural, joining synsets on their glosses (unique within the + * fixtures) and comparing everything else through that join. + */ +public class ReaderEquivalenceTest { + + @Test + void testBothReadersProduceEquivalentViews() { + final InMemoryWordNetLexicon lmf = (InMemoryWordNetLexicon) WnLmfReaderTest.fixture(); + final InMemoryWordNetLexicon wndb = (InMemoryWordNetLexicon) WndbReaderTest.fixture(); + assertEquals(lmf.size(), wndb.size(), "Both fixtures encode the same synsets"); + + final Map wndbByGloss = byGloss(wndb); + assertEquals(byGloss(lmf).keySet(), wndbByGloss.keySet(), "Same glosses on both sides"); + + for (final Synset expected : lmf.synsets()) { + final Synset actual = wndbByGloss.get(expected.gloss()); + assertNotNull(actual, "WNDB view has a synset for gloss: " + expected.gloss()); + assertEquals(expected.pos(), actual.pos(), "Part of speech for: " + expected.gloss()); + assertEquals(expected.lemmas(), actual.lemmas(), "Lemmas for: " + expected.gloss()); + assertEquals(relationsByGloss(expected, lmf), relationsByGloss(actual, wndb), + "Relations for: " + expected.gloss()); + } + } + + @Test + void testLookupAgreesForEveryLemmaAndPos() { + final LexicalKnowledgeBase lmf = WnLmfReaderTest.fixture(); + final InMemoryWordNetLexicon wndb = (InMemoryWordNetLexicon) WndbReaderTest.fixture(); + final Set checked = new HashSet<>(); + for (final Synset synset : wndb.synsets()) { + for (final String lemma : synset.lemmas()) { + if (!checked.add(lemma + "/" + synset.pos())) { + continue; + } + assertEquals( + glosses(lmf.lookup(lemma, synset.pos())), + glosses(wndb.lookup(lemma, synset.pos())), + "Sense sequence for " + lemma + " as " + synset.pos()); + } + } + for (final WordNetPOS pos : WordNetPOS.values()) { + assertEquals(lmf.contains("dog", pos), wndb.contains("dog", pos)); + } + } + + @Test + void testSenseOrderAgreesForMultiSenseLemma() { + final LexicalKnowledgeBase lmf = WnLmfReaderTest.fixture(); + final LexicalKnowledgeBase wndb = WndbReaderTest.fixture(); + final List lmfOrder = glosses(lmf.lookup("run", WordNetPOS.NOUN)); + final List wndbOrder = glosses(wndb.lookup("run", WordNetPOS.NOUN)); + assertEquals(2, lmfOrder.size()); + assertEquals(lmfOrder, wndbOrder); + } + + private static Map byGloss(InMemoryWordNetLexicon lexicon) { + final Map byGloss = new HashMap<>(); + for (final Synset synset : lexicon.synsets()) { + final Synset previous = byGloss.put(synset.gloss(), synset); + assertEquals(null, previous, "Fixture glosses must be unique, duplicated: " + + synset.gloss()); + } + return byGloss; + } + + // A synset's relations with targets replaced by their glosses, id-scheme independent. + private static Map> relationsByGloss(Synset synset, + LexicalKnowledgeBase lexicon) { + final Map> result = new HashMap<>(); + for (final Map.Entry> relation : + synset.relations().entrySet()) { + final Set targetGlosses = new HashSet<>(); + for (final String targetId : relation.getValue()) { + targetGlosses.add(lexicon.synset(targetId).orElseThrow().gloss()); + } + result.put(relation.getKey(), targetGlosses); + } + return result; + } + + private static List glosses(List synsets) { + return synsets.stream().map(Synset::gloss).toList(); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java new file mode 100644 index 0000000000..5d48d17969 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class WnLmfReaderTest { + + static LexicalKnowledgeBase fixture() { + try (InputStream in = WnLmfReaderTest.class.getResourceAsStream("mini-wn-lmf.xml")) { + assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath"); + return WnLmfReader.read(in, "mini-wn-lmf.xml"); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException from a classpath stream", e); + } + } + + private static LexicalKnowledgeBase parse(String document) throws IOException { + return WnLmfReader.read( + new ByteArrayInputStream(document.getBytes(StandardCharsets.UTF_8)), "inline.xml"); + } + + private static String wrap(String body) { + return "\n\n" + + "\n" + + body + "\n\n\n"; + } + + @Test + void testLookupReturnsSynsetWithAllComponents() { + final List senses = fixture().lookup("dog", WordNetPOS.NOUN); + assertEquals(1, senses.size()); + final Synset dog = senses.get(0); + assertEquals("mini-n1", dog.id()); + assertEquals(WordNetPOS.NOUN, dog.pos()); + assertEquals(List.of("dog", "domestic dog"), dog.lemmas()); + assertEquals("a domesticated canid", dog.gloss()); + assertEquals(List.of("mini-n2"), dog.related(WordNetRelation.HYPERNYM)); + } + + @Test + void testLookupFoldsCaseAndUnderscore() { + final LexicalKnowledgeBase lexicon = fixture(); + assertEquals("mini-n1", lexicon.lookup("Domestic_Dog", WordNetPOS.NOUN).get(0).id()); + assertEquals("mini-n1", lexicon.lookup("DOG", WordNetPOS.NOUN).get(0).id()); + } + + @Test + void testLookupKeepsSenseOrder() { + final List runSenses = fixture().lookup("run", WordNetPOS.NOUN); + assertEquals(List.of("mini-n5", "mini-n9"), + runSenses.stream().map(Synset::id).toList()); + } + + @Test + void testLookupIsPosScoped() { + final LexicalKnowledgeBase lexicon = fixture(); + assertEquals(1, lexicon.lookup("run", WordNetPOS.VERB).size()); + assertTrue(lexicon.lookup("dog", WordNetPOS.VERB).isEmpty()); + assertFalse(lexicon.contains("walk", WordNetPOS.NOUN)); + assertTrue(lexicon.contains("walk", WordNetPOS.VERB)); + } + + @Test + void testRelationNavigation() { + final LexicalKnowledgeBase lexicon = fixture(); + assertEquals(List.of("mini-n1"), lexicon.related("mini-n2", WordNetRelation.HYPONYM)); + assertEquals(List.of("mini-v1", "mini-v2"), + lexicon.related("mini-v4", WordNetRelation.HYPONYM)); + assertEquals(List.of("mini-v4"), lexicon.related("mini-v1", WordNetRelation.HYPERNYM)); + } + + @Test + void testRelationTargetSharesCanonicalIdInstance() { + final LexicalKnowledgeBase lexicon = fixture(); + final String target = lexicon.synset("mini-n1").orElseThrow() + .related(WordNetRelation.HYPERNYM).get(0); + // Not just equal: the identical instance from the synset table, so a loaded lexicon keeps + // one copy of each id no matter how many relations point at it. + assertSame(lexicon.synset("mini-n2").orElseThrow().id(), target); + } + + @Test + void testSenseRelationsAreLiftedToSynsetLevel() { + final LexicalKnowledgeBase lexicon = fixture(); + assertEquals(List.of("mini-a2"), lexicon.related("mini-a1", WordNetRelation.ANTONYM)); + assertEquals(List.of("mini-a1"), lexicon.related("mini-a2", WordNetRelation.ANTONYM)); + assertEquals(List.of("mini-v1"), + lexicon.related("mini-n5", WordNetRelation.DERIVATIONALLY_RELATED)); + assertEquals(List.of("mini-n5"), + lexicon.related("mini-v1", WordNetRelation.DERIVATIONALLY_RELATED)); + } + + @Test + void testSatelliteNormalizesToAdjective() { + final List senses = fixture().lookup("large", WordNetPOS.ADJECTIVE); + assertEquals(1, senses.size()); + assertEquals(WordNetPOS.ADJECTIVE, senses.get(0).pos()); + assertEquals(List.of("mini-a4"), fixture().related("mini-a3", WordNetRelation.SIMILAR_TO)); + assertEquals(List.of("mini-a3"), fixture().related("mini-a4", WordNetRelation.SIMILAR_TO)); + } + + @Test + void testSimilarOnVerbSynsetMapsToVerbGroup() throws IOException { + // Documents derived from Princeton data express verb groups as similar on verb synsets; + // the fixture only carries similar on adjectives, so this pins the verb branch directly. + final LexicalKnowledgeBase lexicon = parse(wrap( + "" + + "" + + "" + + "" + + "" + + "produce musical tones" + + "" + + "" + + "sing monotonously")); + assertEquals(List.of("t-v2"), lexicon.related("t-v1", WordNetRelation.VERB_GROUP)); + assertTrue(lexicon.related("t-v1", WordNetRelation.SIMILAR_TO).isEmpty()); + } + + @Test + void testUnknownLemmaOrSynsetIsEmpty() { + final LexicalKnowledgeBase lexicon = fixture(); + assertTrue(lexicon.lookup("zebra", WordNetPOS.NOUN).isEmpty()); + assertTrue(lexicon.synset("mini-n99").isEmpty()); + } + + @Test + void testReadPath(@TempDir Path tempDir) throws IOException { + final Path file = tempDir.resolve("tiny.xml"); + Files.writeString(file, wrap( + "" + + "" + + "a feline")); + final LexicalKnowledgeBase lexicon = WnLmfReader.read(file); + assertEquals("a feline", lexicon.lookup("cat", WordNetPOS.NOUN).get(0).gloss()); + } + + @Test + void testReadPathRejectsNullAndMissing(@TempDir Path tempDir) { + assertThrows(IllegalArgumentException.class, () -> WnLmfReader.read((Path) null)); + assertThrows(IllegalArgumentException.class, + () -> WnLmfReader.read(tempDir.resolve("absent.xml"))); + } + + @Test + void testReadStreamRejectsNulls() { + assertThrows(IllegalArgumentException.class, () -> WnLmfReader.read(null, "x")); + assertThrows(IllegalArgumentException.class, + () -> WnLmfReader.read(new ByteArrayInputStream(new byte[0]), null)); + } + + @Test + void testStreamReadFailurePropagatesAsIOException() { + final InputStream failing = new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("Simulated stream failure"); + } + }; + final IOException e = + assertThrows(IOException.class, () -> WnLmfReader.read(failing, "failing.xml")); + // The I/O failure must surface as itself, not be misreported as a malformed document. + assertFalse(e instanceof InvalidFormatException); + } + + @Test + void testSkipsDoctypeDeclaration() throws IOException { + // Real Open English WordNet releases ship exactly this shape: a DOCTYPE naming the schema + // DTD by an unreachable SYSTEM identifier (example.invalid is the RFC 2606 reserved domain + // that must never resolve). The reader must parse past it without attempting to fetch it. + final String document = "\n" + + "\n" + + "" + + "" + + "" + + "a feline" + + ""; + final LexicalKnowledgeBase lexicon = parse(document); + assertEquals("a feline", lexicon.lookup("cat", WordNetPOS.NOUN).get(0).gloss()); + } + + @Test + void testInternalSubsetEntityIsNeverExpanded(@TempDir Path tempDir) throws IOException { + // A DOCTYPE-declared internal-subset entity is the classic XXE payload: if the parser ever + // honored it, the entity reference below would be replaced by the target file's content. + // With SUPPORT_DTD disabled the declaration itself is never registered, so the reference is + // undefined and parsing must fail loud rather than silently expand it. + final Path secret = tempDir.resolve("secret.txt"); + Files.writeString(secret, "xxe-marker-should-never-appear"); + final String document = "\n" + + "]>\n" + + "" + + "" + + "" + + "a feline" + + ""; + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> parse(document)); + assertFalse(e.getMessage().contains("xxe-marker-should-never-appear")); + } + + @Test + void testRejectsTruncatedDocument() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> parse("\n parse( + wrap("" + + ""))); + assertTrue(e.getMessage().contains("synset")); + } + + @Test + void testRejectsSenseToUndeclaredSynset() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + ""))); + assertTrue(e.getMessage().contains("t-9")); + } + + @Test + void testRejectsRelationToUndeclaredSynset() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline" + + ""))); + assertTrue(e.getMessage().contains("t-9")); + } + + @Test + void testRejectsUnknownRelationType() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline" + + ""))); + assertTrue(e.getMessage().contains("quasi_synonym")); + } + + @Test + void testSkipsOtherRelationTypeOnSenseRelation() throws IOException { + final LexicalKnowledgeBase lexicon = parse( + wrap("" + + "" + + "" + + "a feline")); + assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty()); + } + + @Test + void testSkipsOtherRelationTypeOnSynsetRelation() throws IOException { + // The DTD permits relType="other" on SynsetRelation too, and several OMW-family wordnets + // emit it; it is skipped exactly like the SenseRelation case, not rejected. + final LexicalKnowledgeBase lexicon = parse( + wrap("" + + "" + + "a feline" + + "")); + assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty()); + } + + @Test + void testRejectsUnknownPartOfSpeech() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("x")); + } + + @Test + void testRejectsSynsetWithoutMembers() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("orphan"))); + assertTrue(e.getMessage().contains("t-1")); + } + + @Test + void testRejectsDuplicateSynsetId() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline" + + "a repeat"))); + assertTrue(e.getMessage().contains("Duplicate synset id t-1")); + } + + @Test + void testRejectsDuplicateLexicalEntryId() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("Duplicate lexical entry id t-cat-n")); + } + + @Test + void testRejectsDuplicateSenseId() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "a feline" + + "a second"))); + assertTrue(e.getMessage().contains("Duplicate sense id t-cat-n-1")); + } + + @Test + void testRejectsSynsetMemberPosMismatch() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("t-cat-n")); + assertTrue(e.getMessage().contains("VERB")); + assertTrue(e.getMessage().contains("NOUN")); + } + + @Test + void testRejectsSenseRelationToUndeclaredSense() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("t-ghost-1")); + } + + @Test + void testRejectsLemmaOutsideLexicalEntry() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> parse(wrap(""))); + assertTrue(e.getMessage().contains("Lemma outside a LexicalEntry")); + } + + @Test + void testRejectsSenseBeforeLemma() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("Sense before its entry's Lemma")); + } + + @Test + void testRejectsSenseRelationOutsideSense() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("SenseRelation outside a Sense")); + } + + @Test + void testRejectsSynsetRelationOutsideSynset() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> parse(wrap(""))); + assertTrue(e.getMessage().contains("SynsetRelation outside a Synset")); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java new file mode 100644 index 0000000000..5931957e3b --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; +import java.util.function.UnaryOperator; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class WndbReaderTest { + + private static final String DOG_ID = "wndb-00001075-n"; + private static final String CANID_ID = "wndb-00001160-n"; + + static Path fixtureDirectory() { + final URL url = WndbReaderTest.class.getResource("mini-wndb"); + assertNotNull(url, "Fixture directory mini-wndb must be on the test classpath"); + try { + return Path.of(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException("Unexpected fixture URI: " + url, e); + } + } + + static LexicalKnowledgeBase fixture() { + try { + return WndbReader.read(fixtureDirectory()); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException reading the WNDB fixture", e); + } + } + + @Test + void testLookupReturnsSynsetWithAllComponents() { + final List senses = fixture().lookup("dog", WordNetPOS.NOUN); + assertEquals(1, senses.size()); + final Synset dog = senses.get(0); + assertEquals(DOG_ID, dog.id()); + assertEquals(WordNetPOS.NOUN, dog.pos()); + assertEquals(List.of("dog", "domestic dog"), dog.lemmas()); + assertEquals("a domesticated canid", dog.gloss()); + assertEquals(List.of(CANID_ID), dog.related(WordNetRelation.HYPERNYM)); + } + + @Test + void testLookupFoldsCaseAndUnderscore() { + final LexicalKnowledgeBase lexicon = fixture(); + assertEquals(DOG_ID, lexicon.lookup("Domestic_Dog", WordNetPOS.NOUN).get(0).id()); + assertEquals(DOG_ID, lexicon.lookup("DOG", WordNetPOS.NOUN).get(0).id()); + } + + @Test + void testLookupKeepsIndexSenseOrder() { + assertEquals(List.of("wndb-00001427-n", "wndb-00001669-n"), + fixture().lookup("run", WordNetPOS.NOUN).stream().map(Synset::id).toList()); + } + + @Test + void testRelationNavigation() { + final LexicalKnowledgeBase lexicon = fixture(); + assertEquals(List.of(DOG_ID), lexicon.related(CANID_ID, WordNetRelation.HYPONYM)); + assertEquals(List.of("wndb-00001075-v", "wndb-00001171-v"), + lexicon.related("wndb-00001324-v", WordNetRelation.HYPONYM)); + assertEquals(List.of("wndb-00001075-v"), + lexicon.related("wndb-00001427-n", WordNetRelation.DERIVATIONALLY_RELATED)); + } + + @Test + void testRelationTargetSharesCanonicalIdInstance() { + final LexicalKnowledgeBase lexicon = fixture(); + final String target = lexicon.synset(DOG_ID).orElseThrow() + .related(WordNetRelation.HYPERNYM).get(0); + // Not just equal: the identical instance from the synset table, so a loaded lexicon keeps + // one copy of each id no matter how many pointers reference it. + assertSame(lexicon.synset(CANID_ID).orElseThrow().id(), target); + } + + @Test + void testLexicalPointersSurfaceAtSynsetLevel() { + final LexicalKnowledgeBase lexicon = fixture(); + assertEquals(List.of("wndb-00001141-a"), + lexicon.related("wndb-00001075-a", WordNetRelation.ANTONYM)); + assertEquals(List.of("wndb-00001075-a"), + lexicon.related("wndb-00001141-a", WordNetRelation.ANTONYM)); + } + + @Test + void testSatelliteNormalizesToAdjectiveAndMarkerIsStripped() { + final LexicalKnowledgeBase lexicon = fixture(); + final Synset large = lexicon.lookup("large", WordNetPOS.ADJECTIVE).get(0); + assertEquals(WordNetPOS.ADJECTIVE, large.pos()); + assertEquals(List.of("wndb-00001211-a"), large.related(WordNetRelation.SIMILAR_TO)); + // short is stored as short(p); the syntactic marker is not part of the lemma. + assertEquals(List.of("short"), + lexicon.lookup("short", WordNetPOS.ADJECTIVE).get(0).lemmas()); + } + + @Test + void testVerbGroupPointerMapsToVerbGroup(@TempDir Path tempDir) throws IOException { + // The fixture has no $ pointer, so the VERB_GROUP mapping is pinned against a minimal + // constructed database whose byte offsets are computed, not hard-coded: every offset field + // is exactly eight digits, so the second line's position is independent of the digit values. + writeEmptyDb(tempDir, "noun", "adj", "adv"); + final String template = + "00000000 29 v 01 sing 0 001 $ XXXXXXXX v 0000 00 | produce musical tones"; + final String off2 = String.format(Locale.ROOT, "%08d", template.length() + 1); + final String line1 = template.replace("XXXXXXXX", off2); + final String line2 = off2 + " 29 v 01 chant 0 001 $ 00000000 v 0000 00 | sing monotonously"; + Files.writeString(tempDir.resolve("data.verb"), line1 + "\n" + line2 + "\n", + StandardCharsets.ISO_8859_1); + Files.writeString(tempDir.resolve("index.verb"), + "chant v 1 1 $ 1 0 " + off2 + "\nsing v 1 1 $ 1 0 00000000\n", + StandardCharsets.ISO_8859_1); + final LexicalKnowledgeBase lexicon = WndbReader.read(tempDir); + assertEquals(List.of("wndb-" + off2 + "-v"), + lexicon.related("wndb-00000000-v", WordNetRelation.VERB_GROUP)); + assertEquals(List.of("wndb-00000000-v"), + lexicon.related("wndb-" + off2 + "-v", WordNetRelation.VERB_GROUP)); + } + + @Test + void testUnknownLemmaOrSynsetIsEmpty() { + final LexicalKnowledgeBase lexicon = fixture(); + assertTrue(lexicon.lookup("zebra", WordNetPOS.NOUN).isEmpty()); + assertTrue(lexicon.synset("wndb-99999999-n").isEmpty()); + } + + @Test + void testRejectsNullAndMissingDirectory(@TempDir Path tempDir) { + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(null)); + assertThrows(IllegalArgumentException.class, + () -> WndbReader.read(tempDir.resolve("absent"))); + } + + @Test + void testRejectsMissingDatabaseFile(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + Files.delete(tempDir.resolve("data.verb")); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("data.verb")); + } + + @Test + void testRejectsIndexOffsetWithoutDataLine(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "index.noun", line -> line.startsWith("berry ") + ? line.replace("00001564", "00001565") : line); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("berry")); + assertTrue(e.getMessage().contains("00001565")); + } + + @Test + void testRejectsDataOffsetFieldMismatch(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", + line -> line.replace("00001503 03 n 01 box", "00001504 03 n 01 box")); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("disagrees")); + } + + @Test + void testRejectsTruncatedDataLine(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", line -> line.startsWith("00001564") + ? line.substring(0, line.indexOf(" 000 |")) : line); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("data.noun")); + assertTrue(e.getMessage().contains("Truncated")); + } + + @Test + void testRejectsUndeclaredPointerSymbol(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000", + "001 ? 00001160 n 0000")); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("Undeclared pointer symbol: ?")); + } + + @Test + void testRejectsPointerToNonexistentSynset(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000", + "001 @ 00009999 n 0000")); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("wndb-00009999-n")); + } + + @Test + void testDanglingPointerErrorNamesPointerLine(@TempDir Path tempDir) throws IOException { + // A constructed database with no preamble, so the dangling pointer sits on a known line + // and the error message can be pinned to name it. + writeEmptyDb(tempDir, "noun", "adj", "adv"); + Files.writeString(tempDir.resolve("data.verb"), + "00000000 29 v 01 sing 0 001 $ 00009999 v 0000 00 | produce musical tones\n", + StandardCharsets.ISO_8859_1); + Files.writeString(tempDir.resolve("index.verb"), "sing v 1 1 $ 1 0 00000000\n", + StandardCharsets.ISO_8859_1); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("wndb-00009999-v")); + assertTrue(e.getMessage().contains("line 1")); + } + + private static void writeEmptyDb(Path directory, String... suffixes) throws IOException { + for (final String suffix : suffixes) { + Files.writeString(directory.resolve("data." + suffix), ""); + Files.writeString(directory.resolve("index." + suffix), ""); + } + } + + private static void copyFixture(Path target) throws IOException { + try (var files = Files.list(fixtureDirectory())) { + for (final Path file : files.toList()) { + Files.copy(file, target.resolve(file.getFileName().toString())); + } + } + } + + // Applies a line transformation to one fixture file. The mutations only ever keep or shrink + // line lengths of the affected line's own fields, so surrounding offsets stay valid. + private static void mutate(Path directory, String fileName, UnaryOperator edit) + throws IOException { + final Path file = directory.resolve(fileName); + final List lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1); + final StringBuilder out = new StringBuilder(); + for (final String line : lines) { + out.append(edit.apply(line)).append('\n'); + } + Files.writeString(file, out.toString(), StandardCharsets.ISO_8859_1); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml new file mode 100644 index 0000000000..10e039214b --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml @@ -0,0 +1,183 @@ + + + + + + + dog + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + a domesticated canid + + the dog barked + + + a carnivorous mammal with nonretractile claws + + + + a small rodent with a long tail + + + + a gnawing mammal with chisel teeth + + + + an act of running at speed + + + a rigid rectangular container + + + a small juicy fruit + + + an adult male person + + + a score made in baseball + + + move fast on foot + + + + move at a regular pace + + + + change location or position + + + change position in space + + + + + of great height + + + of small height + + + of great size + + + + above average in size + + + + with speed + + + in a good or proper manner + + + diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes new file mode 100644 index 0000000000..3868b4d103 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes @@ -0,0 +1,7 @@ +# WNDB is a byte-offset format: each data line embeds its own byte position in the +# file, and WndbReader validates that offset against the actual position. Line-ending +# normalization on checkout (the repo root's `* text=auto`) would insert a CR before +# every LF on Windows, shifting every offset after the first line and breaking every +# fixture that has more than one line. Disable it here so checkout is byte-identical +# on every platform. +* -text diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc new file mode 100644 index 0000000000..404a2e4ddb --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc @@ -0,0 +1 @@ +better good diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc new file mode 100644 index 0000000000..c43a2cd529 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc @@ -0,0 +1 @@ +best well diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj new file mode 100644 index 0000000000..1340d16a45 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 00 a 01 tall 0 001 ! 00001141 a 0101 | of great height +00001141 00 a 01 short(p) 0 001 ! 00001075 a 0101 | of small height +00001211 00 a 01 big 0 001 & 00001274 a 0000 | of great size +00001274 00 s 01 large 0 001 & 00001211 a 0000 | above average in size diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv new file mode 100644 index 0000000000..732c21dfd5 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv @@ -0,0 +1,20 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 02 r 01 quickly 0 000 | with speed +00001121 02 r 01 well 0 000 | in a good or proper manner diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun new file mode 100644 index 0000000000..0598111bf1 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun @@ -0,0 +1,27 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 03 n 02 dog 0 domestic_dog 0 001 @ 00001160 n 0000 | a domesticated canid +00001160 03 n 01 canid 0 001 ~ 00001075 n 0000 | a carnivorous mammal with nonretractile claws +00001257 03 n 01 mouse 0 001 @ 00001340 n 0000 | a small rodent with a long tail +00001340 03 n 01 rodent 0 001 ~ 00001257 n 0000 | a gnawing mammal with chisel teeth +00001427 03 n 01 run 0 001 + 00001075 v 0101 | an act of running at speed +00001503 03 n 01 box 0 000 | a rigid rectangular container +00001564 03 n 01 berry 0 000 | a small juicy fruit +00001617 03 n 01 man 0 000 | an adult male person +00001669 03 n 01 run 0 000 | a score made in baseball diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb new file mode 100644 index 0000000000..048546ed71 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 29 v 01 run 0 002 @ 00001324 v 0000 + 00001427 n 0101 01 + 02 00 | move fast on foot +00001171 29 v 01 walk 0 001 @ 00001324 v 0000 01 + 02 00 | move at a regular pace +00001255 29 v 01 go 0 000 01 + 02 00 | change location or position +00001324 29 v 01 move 0 002 ~ 00001075 v 0000 ~ 00001171 v 0000 01 + 02 00 | change position in space diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj new file mode 100644 index 0000000000..827a988a7d --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +big a 1 1 & 1 0 00001211 +large a 1 1 & 1 0 00001274 +short a 1 1 ! 1 0 00001141 +tall a 1 1 ! 1 0 00001075 diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv new file mode 100644 index 0000000000..da20fe1193 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv @@ -0,0 +1,20 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +quickly r 1 0 1 0 00001075 +well r 1 0 1 0 00001121 diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun new file mode 100644 index 0000000000..41a8a4317b --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun @@ -0,0 +1,27 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +berry n 1 0 1 0 00001564 +box n 1 0 1 0 00001503 +canid n 1 1 ~ 1 0 00001160 +dog n 1 1 @ 1 0 00001075 +domestic_dog n 1 1 @ 1 0 00001075 +man n 1 0 1 0 00001617 +mouse n 1 1 @ 1 0 00001257 +rodent n 1 1 ~ 1 0 00001340 +run n 2 1 + 2 1 00001427 00001669 diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb new file mode 100644 index 0000000000..2b380478de --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +go v 1 0 1 0 00001255 +move v 1 1 ~ 1 0 00001324 +run v 1 2 @ + 1 1 00001075 +walk v 1 1 @ 1 0 00001171 diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc new file mode 100644 index 0000000000..e5b3080a88 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc @@ -0,0 +1,3 @@ +men man +mice mouse +oxen ox diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc new file mode 100644 index 0000000000..486d0c7851 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc @@ -0,0 +1,4 @@ +gone go +ran run +running run +went go diff --git a/opennlp-extensions/pom.xml b/opennlp-extensions/pom.xml index 9afcd3fe3c..8c0e432a58 100644 --- a/opennlp-extensions/pom.xml +++ b/opennlp-extensions/pom.xml @@ -41,6 +41,7 @@ opennlp-morfologik opennlp-spellcheck opennlp-uima + opennlp-wordnet \ No newline at end of file diff --git a/pom.xml b/pom.xml index 53aa93d56d..6c1863811a 100644 --- a/pom.xml +++ b/pom.xml @@ -216,6 +216,12 @@ ${project.version} + + opennlp-wordnet + ${project.groupId} + ${project.version} + + opennlp-uima ${project.groupId} diff --git a/rat-excludes b/rat-excludes index 5a5d86b90c..561869bd46 100644 --- a/rat-excludes +++ b/rat-excludes @@ -70,3 +70,12 @@ src/main/resources/opennlp/tools/tokenize/uax29/WordBreakProperty.txt src/main/resources/opennlp/tools/tokenize/uax29/ExtendedPictographic.txt src/main/resources/opennlp/tools/util/normalizer/confusables.txt src/test/resources/opennlp/tools/tokenize/uax29/WordBreakTest.txt + + +src/test/resources/opennlp/wordnet/mini-wndb/noun.exc +src/test/resources/opennlp/wordnet/mini-wndb/verb.exc +src/test/resources/opennlp/wordnet/mini-wndb/adj.exc +src/test/resources/opennlp/wordnet/mini-wndb/adv.exc From e08bf29c284103ee5fa113f051f9aaba486d6a60 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:40:03 -0400 Subject: [PATCH 02/15] OPENNLP-1880: Add a WordNet manual chapter with a mirror-tested example Add docbkx/wordnet.xml, wire it into the manual, and add WordNetUsageExampleTest asserting the lookups the chapter prints. --- opennlp-docs/src/docbkx/opennlp.xml | 1 + opennlp-docs/src/docbkx/wordnet.xml | 82 +++++++++++++++++++ .../wordnet/WordNetUsageExampleTest.java | 78 ++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 opennlp-docs/src/docbkx/wordnet.xml create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java diff --git a/opennlp-docs/src/docbkx/opennlp.xml b/opennlp-docs/src/docbkx/opennlp.xml index 36641c2c89..f7a8a39406 100644 --- a/opennlp-docs/src/docbkx/opennlp.xml +++ b/opennlp-docs/src/docbkx/opennlp.xml @@ -109,6 +109,7 @@ under the License. + diff --git a/opennlp-docs/src/docbkx/wordnet.xml b/opennlp-docs/src/docbkx/wordnet.xml new file mode 100644 index 0000000000..7b01e99860 --- /dev/null +++ b/opennlp-docs/src/docbkx/wordnet.xml @@ -0,0 +1,82 @@ + + + + + + + WordNet + +
+ Introduction + + The opennlp-wordnet module loads a WordNet-style lexicon into + a LexicalKnowledgeBase and looks up synsets by lemma and part + of speech. Two readers are provided: WnLmfReader for the + Global WordNet Association WN-LMF XML interchange format, and + WndbReader for the classic Princeton WordNet database file + layout. Both return an immutable, thread-safe knowledge base. + +
+ +
+ Loading a lexicon + + WN-LMF is the usual choice for Open English WordNet and other GWA + wordnets. WNDB remains available for a local Princeton-style + dict directory. The examples below use the miniature + fixtures from the module's tests; replace the paths with a full lexicon + in application code. WordNetUsageExampleTest asserts the + behavior shown here. + + + +
+ +
+ Lookup + + Lookups are scoped by part of speech and fold case and underscores the + same way the readers index lemmas. Against the miniature WN-LMF fixture, + the noun dog has one sense: + senses = lexicon.lookup("dog", WordNetPOS.NOUN); +// senses.size() = 1 +// senses.get(0).id() = "mini-n1" +// senses.get(0).lemmas() = ["dog", "domestic dog"] +// senses.get(0).gloss() = "a domesticated canid"]]> + + +
+ +
+ Morphy lemmatization + + MorphyLemmatizer implements the Morphy algorithm against a + loaded lexicon and the irregular-form exception lists + (noun.exc, verb.exc, adj.exc, + adv.exc). Exception hits are returned first; regular + detachments are kept only when the candidate is in the lexicon. Unknown + forms yield the marker O. + + + +
+ diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java new file mode 100644 index 0000000000..530e25ce66 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.wordnet; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; + +/** + * Runs the manual's WordNet examples (docbkx {@code wordnet.xml}) verbatim: every value + * the chapter states is asserted here, so a change breaking this test breaks the manual. + * The lexicon is the classpath fixture {@code mini-wn-lmf.xml}; exception lists come from + * the sibling {@code mini-wndb} directory. + */ +public class WordNetUsageExampleTest { + + private static LexicalKnowledgeBase loadMiniWnLmf() throws IOException { + try (InputStream in = WordNetUsageExampleTest.class.getResourceAsStream("mini-wn-lmf.xml")) { + Assertions.assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath"); + return WnLmfReader.read(in, "mini-wn-lmf.xml"); + } + } + + private static Path miniWndbDirectory() { + final URL url = WordNetUsageExampleTest.class.getResource("mini-wndb"); + Assertions.assertNotNull(url, "Fixture directory mini-wndb must be on the test classpath"); + try { + return Path.of(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException("Unexpected fixture URI: " + url, e); + } + } + + /** + * Load, lookup, and Morphy lemmatize as the chapter shows. + */ + @Test + void testLoadLookupAndLemmatize() throws IOException { + final LexicalKnowledgeBase lexicon = loadMiniWnLmf(); + final List senses = lexicon.lookup("dog", WordNetPOS.NOUN); + Assertions.assertEquals(1, senses.size()); + Assertions.assertEquals("mini-n1", senses.get(0).id()); + Assertions.assertEquals(List.of("dog", "domestic dog"), senses.get(0).lemmas()); + Assertions.assertEquals("a domesticated canid", senses.get(0).gloss()); + + final MorphyLemmatizer lemmatizer = + new MorphyLemmatizer(lexicon, MorphyExceptions.load(miniWndbDirectory())); + Assertions.assertEquals("mouse", + lemmatizer.lemmatize(new String[] {"mice"}, new String[] {"NNS"})[0]); + Assertions.assertEquals("dog", + lemmatizer.lemmatize(new String[] {"dogs"}, new String[] {"NNS"})[0]); + } +} From 22beb194658cf437fa3506b3d795e54709c28592 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 06:48:00 -0400 Subject: [PATCH 03/15] OPENNLP-1880: Align null contracts, annotations, and dev helper placement with the review conventions --- .../java/opennlp/wordnet/MorphyLemmatizer.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java index c4da01d94f..a7f61b1db4 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -100,8 +100,11 @@ public MorphyLemmatizer(LexicalKnowledgeBase lexicon, MorphyExceptions exception */ @Override public String[] lemmatize(String[] toks, String[] tags) { - if (toks == null || tags == null) { - throw new IllegalArgumentException("Toks and tags must not be null"); + if (toks == null) { + throw new IllegalArgumentException("Toks must not be null"); + } + if (tags == null) { + throw new IllegalArgumentException("Tags must not be null"); } if (toks.length != tags.length) { throw new IllegalArgumentException("Toks and tags must have the same length, got " @@ -123,8 +126,11 @@ public String[] lemmatize(String[] toks, String[] tags) { */ @Override public List> lemmatize(List toks, List tags) { - if (toks == null || tags == null) { - throw new IllegalArgumentException("Toks and tags must not be null"); + if (toks == null) { + throw new IllegalArgumentException("Toks must not be null"); + } + if (tags == null) { + throw new IllegalArgumentException("Tags must not be null"); } if (toks.size() != tags.size()) { throw new IllegalArgumentException("Toks and tags must have the same size, got " From b2b043c477aa78eca4273f3efc72496982f34ebd Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:01:20 -0400 Subject: [PATCH 04/15] OPENNLP-1880: Address review: narrow the contract, validate at the boundary, pin WNDB rejections - Narrow the LexicalKnowledgeBase javadoc so the interface stops prescribing what only an implementation can promise: lemma matching semantics and thread safety are now stated as implementation specific and documented where they hold, on InMemoryWordNetLexicon, which already carries @ThreadSafe and describes the folding it applies. - Reword the contains() javadoc to say plainly that the default implementation delegates to lookup(), instead of speculating about cheaper overrides. - Move the null-element checks in MorphyLemmatizer up to the public lemmatize() overloads, both the array form and the list form, so validation happens once at the boundary the caller sees; the private lemmasOf() no longer repeats them and now documents that its arguments are validated by the caller. - Reject a null argument in LemmaFolding.splitOnSpaces() rather than letting it fail later as a NullPointerException, and capitalize the fold() message so it matches the wording the other validators use. - Document the throws clauses that the explicit validation adds, on LemmaKey.of() and on splitOnSpaces(). - Extract the repeated WN-LMF attribute names into ID_ATTRIBUTE, PART_OF_SPEECH_ATTRIBUTE, REL_TYPE_ATTRIBUTE, and TARGET_ATTRIBUTE, and the shared error opening into MALFORMED_PREFIX, so the element handlers stop repeating string literals. - Extract the WNDB offset failure detail into MALFORMED_OFFSET, shared by the length check and the digit check. - Fold the duplicated WNDB message construction into malformedMessage(), so the tokenizer builds the text directly instead of constructing an InvalidFormatException only to read getMessage() back off it. - Drop the redundant fileName parameter from WndbReader.readAll(), which already names the full path it failed to open. - Reduce the visibility of the Parser helpers in WnLmfReader: malformed() and line() are now private instance methods like every other helper in that class. - Make MorphyLemmatizer.rulesFor() an instance method for the same reason, so the lemmatizer's private helpers are consistent. - Add the missing javadoc on the RELATION_NAMES and POINTER_SYMBOLS lookup tables and on both RawSynset holders, the last undocumented members in the readers. - Correct two stale comments: the build() comment now points at memberLemmas(), where the synset and member part-of-speech agreement is really checked, and the mutate() comment in the tests states the actual constraint, that an edit which changes a line's length is only safe when the reader is expected to fail on that line before it reads the ones after it. - Add a parameterized WNDB test pinning eight field-level rejections that had no coverage: the offset length and digit checks, the synset and index part of speech mismatches, the base-16 word count field, the minimum word count, the pointer pos, the gloss separator, and the syntactic marker. - Add pinning tests for the newly explicit validation: splitOnSpaces() on null, and the list lemmatize() overload with a null token and with a null tag. - Share the fixtures instead of duplicating them: WndbReaderTest now exposes DOG_ID, CANID_ID, and its fixtureDirectory(), WnLmfReaderTest exposes fixture(), and LexiconConcurrencyTest and WordNetUsageExampleTest use those instead of their own loader copies and hardcoded ids. - Document the package-private test fixture helpers and switch WordNetUsageExampleTest to static assertion imports, matching the other tests in the module. --- .../tools/wordnet/LexicalKnowledgeBase.java | 16 +++---- .../wordnet/InMemoryWordNetLexicon.java | 1 + .../java/opennlp/wordnet/LemmaFolding.java | 6 ++- .../opennlp/wordnet/MorphyLemmatizer.java | 21 ++++++--- .../java/opennlp/wordnet/WnLmfReader.java | 46 +++++++++++++------ .../main/java/opennlp/wordnet/WndbReader.java | 37 ++++++++++----- .../opennlp/wordnet/LemmaFoldingTest.java | 5 ++ .../wordnet/LexiconConcurrencyTest.java | 6 +-- .../opennlp/wordnet/MorphyExceptionsTest.java | 5 ++ .../opennlp/wordnet/MorphyLemmatizerTest.java | 6 +++ .../java/opennlp/wordnet/WnLmfReaderTest.java | 5 ++ .../java/opennlp/wordnet/WndbReaderTest.java | 46 +++++++++++++++++-- .../wordnet/WordNetUsageExampleTest.java | 45 +++++------------- 13 files changed, 163 insertions(+), 82 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java index 214ef74db1..5723219dd2 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java @@ -24,14 +24,11 @@ * identity is opaque and source-qualified (see {@link Synset#id()}). Lookups return their matches * in the source's sense order and never return {@code null}. * - *

Lemma matching semantics are the implementation's concern. The reference implementations - * match case-insensitively (case folding with the root locale) and treat the underscore some - * formats store in multiword lemmas as a space; an implementation with different semantics must - * document them. Returned {@link Synset#lemmas() lemmas} preserve the source's written forms, - * with spaces in multiword lemmas.

+ *

How a queried lemma is matched against the source's written forms is implementation + * specific and documented there. Returned {@link Synset#lemmas() lemmas} preserve the source's + * written forms, with spaces in multiword lemmas.

* - *

Implementations must be immutable and thread-safe after loading: one instance is meant to - * be shared across an application's threads for concurrent lookups.

+ *

Thread safety is implementation specific.

*/ public interface LexicalKnowledgeBase { @@ -74,9 +71,8 @@ default List related(String synsetId, WordNetRelation relation) { } /** - * Tests whether the lexicon contains a lemma with a part of speech. This is the membership - * check morphological rules validate their candidates against; implementations may override - * it with a cheaper check than {@link #lookup(String, WordNetPOS)}. + * Tests whether the lexicon contains a lemma with a part of speech. The default implementation + * delegates to {@link #lookup(String, WordNetPOS)}. * * @param lemma The lemma to test. Must not be {@code null}. * @param pos The part of speech to test it as. Must not be {@code null}. diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java index 2401bfab6a..c5ea32d3c7 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java @@ -148,6 +148,7 @@ record LemmaKey(String lemma, WordNetPOS pos) { * @param writtenForm The lemma as written in the source or query. Must not be {@code null}. * @param pos The part of speech. Must not be {@code null}. * @return The folded key. + * @throws IllegalArgumentException Thrown if {@code writtenForm} is {@code null}. */ static LemmaKey of(String writtenForm, WordNetPOS pos) { return new LemmaKey(LemmaFolding.fold(writtenForm), pos); diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java index 4da713d520..697246731f 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java @@ -41,7 +41,7 @@ private LemmaFolding() { */ static String fold(String writtenForm) { if (writtenForm == null) { - throw new IllegalArgumentException("writtenForm must not be null"); + throw new IllegalArgumentException("WrittenForm must not be null"); } return writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT); } @@ -51,8 +51,12 @@ static String fold(String writtenForm) { * * @param value The field list. Must not be {@code null}. * @return The non-empty fields in order, never {@code null}. + * @throws IllegalArgumentException Thrown if {@code value} is {@code null}. */ static List splitOnSpaces(String value) { + if (value == null) { + throw new IllegalArgumentException("Value must not be null"); + } final List parts = new ArrayList<>(4); int start = 0; while (start < value.length()) { diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java index a7f61b1db4..a73bf70689 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -112,6 +112,12 @@ public String[] lemmatize(String[] toks, String[] tags) { } final String[] lemmas = new String[toks.length]; for (int i = 0; i < toks.length; i++) { + if (toks[i] == null) { + throw new IllegalArgumentException("Toks must not contain a null element"); + } + if (tags[i] == null) { + throw new IllegalArgumentException("Tags must not contain a null element"); + } final List candidates = lemmasOf(toks[i], tags[i]); lemmas[i] = candidates.isEmpty() ? UNKNOWN_LEMMA : candidates.get(0); } @@ -138,6 +144,12 @@ public List> lemmatize(List toks, List tags) { } final List> lemmas = new ArrayList<>(toks.size()); for (int i = 0; i < toks.size(); i++) { + if (toks.get(i) == null) { + throw new IllegalArgumentException("Toks must not contain a null element"); + } + if (tags.get(i) == null) { + throw new IllegalArgumentException("Tags must not contain a null element"); + } final List candidates = lemmasOf(toks.get(i), tags.get(i)); lemmas.add(candidates.isEmpty() ? List.of(UNKNOWN_LEMMA) : candidates); } @@ -147,15 +159,12 @@ public List> lemmatize(List toks, List tags) { /** * Finds all lemmas of one token, most preferred first. * - * @param token The token to lemmatize. - * @param tag The part-of-speech tag. + * @param token The token to lemmatize. Validated at the public boundary. + * @param tag The part-of-speech tag. Validated at the public boundary. * @return The candidate lemmas, empty when the word is unknown or the tag maps to no part of * speech. */ private List lemmasOf(String token, String tag) { - if (token == null || tag == null) { - throw new IllegalArgumentException("Tokens and tags must not contain null elements"); - } final WordNetPOS pos = posFromTag(tag); if (pos == null) { return List.of(); @@ -188,7 +197,7 @@ private List lemmasOf(String token, String tag) { * @param pos The part of speech. * @return The suffix-substitution rules, empty for adverbs. */ - private static String[][] rulesFor(WordNetPOS pos) { + private String[][] rulesFor(WordNetPOS pos) { return switch (pos) { case NOUN -> NOUN_RULES; case VERB -> VERB_RULES; diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java index 1222cfbfd1..fd5cfd73e3 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -66,6 +66,7 @@ */ public final class WnLmfReader { + /** The WN-LMF relation names this reader accepts, mapped to the contract relations. */ private static final Map RELATION_NAMES = relationNames(); /** The format's escape-hatch relation type; carries no type the contract can express. */ @@ -80,6 +81,21 @@ public final class WnLmfReader { /** The element declaring a synset; opened and closed by the same handlers. */ private static final String SYNSET_ELEMENT = "Synset"; + /** The identifier attribute shared by entries, senses, and synsets. */ + private static final String ID_ATTRIBUTE = "id"; + + /** The part-of-speech attribute shared by lemmas and synsets. */ + private static final String PART_OF_SPEECH_ATTRIBUTE = "partOfSpeech"; + + /** The relation-type attribute shared by sense and synset relations. */ + private static final String REL_TYPE_ATTRIBUTE = "relType"; + + /** The relation-target attribute shared by sense and synset relations. */ + private static final String TARGET_ATTRIBUTE = "target"; + + /** The opening of every malformed-document message, before the resource name. */ + private static final String MALFORMED_PREFIX = "Malformed WN-LMF document "; + /** Not instantiable. */ private WnLmfReader() { } @@ -227,7 +243,7 @@ private void startElement(XMLStreamReader reader) final String name = reader.getLocalName(); switch (name) { case LEXICAL_ENTRY_ELEMENT -> { - currentEntryId = requireAttribute(reader, "id"); + currentEntryId = requireAttribute(reader, ID_ATTRIBUTE); if (!entryIds.add(currentEntryId)) { throw malformed(reader.getLocation(), "Duplicate lexical entry id " + currentEntryId, null); @@ -240,7 +256,7 @@ private void startElement(XMLStreamReader reader) throw malformed(reader.getLocation(), "Lemma outside a LexicalEntry", null); } currentEntryLemma = requireAttribute(reader, "writtenForm"); - currentEntryPos = parsePos(requireAttribute(reader, "partOfSpeech"), + currentEntryPos = parsePos(requireAttribute(reader, PART_OF_SPEECH_ATTRIBUTE), reader.getLocation()); lemmaByEntryId.put(currentEntryId, currentEntryLemma); posByEntryId.put(currentEntryId, currentEntryPos); @@ -250,7 +266,7 @@ private void startElement(XMLStreamReader reader) throw malformed(reader.getLocation(), "Sense before its entry's Lemma in LexicalEntry " + currentEntryId, null); } - currentSenseId = requireAttribute(reader, "id"); + currentSenseId = requireAttribute(reader, ID_ATTRIBUTE); final String synsetId = requireAttribute(reader, "synset"); if (synsetBySenseId.putIfAbsent(currentSenseId, synsetId) != null) { throw malformed(reader.getLocation(), "Duplicate sense id " + currentSenseId, null); @@ -269,12 +285,12 @@ private void startElement(XMLStreamReader reader) throw malformed(reader.getLocation(), "SenseRelation outside a Sense", null); } senseRelations.add(new RawSenseRelation(currentSenseId, - requireAttribute(reader, "relType"), requireAttribute(reader, "target"), - line(reader.getLocation()))); + requireAttribute(reader, REL_TYPE_ATTRIBUTE), + requireAttribute(reader, TARGET_ATTRIBUTE), line(reader.getLocation()))); } case SYNSET_ELEMENT -> { - final String id = requireAttribute(reader, "id"); - final WordNetPOS pos = parsePos(requireAttribute(reader, "partOfSpeech"), + final String id = requireAttribute(reader, ID_ATTRIBUTE); + final WordNetPOS pos = parsePos(requireAttribute(reader, PART_OF_SPEECH_ATTRIBUTE), reader.getLocation()); currentSynset = new RawSynset(id, pos, reader.getAttributeValue(null, "members"), line(reader.getLocation())); @@ -291,8 +307,8 @@ private void startElement(XMLStreamReader reader) if (currentSynset == null) { throw malformed(reader.getLocation(), "SynsetRelation outside a Synset", null); } - final String relType = requireAttribute(reader, "relType"); - final String target = requireAttribute(reader, "target"); + final String relType = requireAttribute(reader, REL_TYPE_ATTRIBUTE); + final String target = requireAttribute(reader, TARGET_ATTRIBUTE); // The escape-hatch type is a documented skip, not a rejection. if (!OTHER_RELATION.equals(relType)) { currentSynset.relations.add( @@ -335,7 +351,8 @@ private void endElement(String name) { * target, or a synset has no members. */ LexicalKnowledgeBase build() throws InvalidFormatException { - // Every sense must point to a declared synset, with a consistent part of speech. + // Every sense must point to a declared synset; part-of-speech consistency between a + // synset and its member entries is checked in memberLemmas. for (final Map.Entry sense : synsetBySenseId.entrySet()) { final RawSynset target = rawSynsets.get(sense.getValue()); if (target == null) { @@ -507,10 +524,10 @@ private String requireAttribute(XMLStreamReader reader, String attribute) * @param cause The underlying cause, or {@code null}. * @return The exception to throw. */ - InvalidFormatException malformed(Location location, String message, Throwable cause) { + private InvalidFormatException malformed(Location location, String message, Throwable cause) { final int line = line(location); - final String prefix = line < 0 ? "Malformed WN-LMF document " + resourceName + ": " - : "Malformed WN-LMF document " + resourceName + " at line " + line + ": "; + final String prefix = line < 0 ? MALFORMED_PREFIX + resourceName + ": " + : MALFORMED_PREFIX + resourceName + " at line " + line + ": "; return cause == null ? new InvalidFormatException(prefix + message) : new InvalidFormatException(prefix + message, cause); } @@ -521,11 +538,12 @@ InvalidFormatException malformed(Location location, String message, Throwable ca * @param location The location, or {@code null}. * @return The line number, or {@code -1} when unknown. */ - private static int line(Location location) { + private int line(Location location) { return location == null ? -1 : location.getLineNumber(); } } + /** A parsed synset, kept until its members and relation targets can be resolved. */ private static final class RawSynset { private final String id; private final WordNetPOS pos; diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java index ed6b18ea8b..1fd05fce60 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java @@ -57,11 +57,15 @@ */ public final class WndbReader { + /** The WNDB pointer symbols this reader accepts, mapped to the contract relations. */ private static final Map POINTER_SYMBOLS = pointerSymbols(); /** The prefix of every synset id this reader mints. */ private static final String SYNSET_ID_PREFIX = "wndb-"; + /** The failure detail for a synset offset field that is not exactly 8 digits. */ + private static final String MALFORMED_OFFSET = "Synset offset must be 8 digits, got: "; + /** Not instantiable. */ private WndbReader() { } @@ -146,7 +150,7 @@ private enum FilePos { private static void parseDataFile(Path directory, FilePos filePos, Map rawSynsets) throws IOException { final String fileName = "data." + filePos.suffix; - final byte[] bytes = readAll(directory.resolve(fileName), fileName); + final byte[] bytes = readAll(directory.resolve(fileName)); int lineStart = 0; int lineNumber = 0; while (lineStart < bytes.length) { @@ -281,7 +285,7 @@ private static void parseIndexFile(Path directory, FilePos filePos, Map> senses) throws IOException { final String fileName = "index." + filePos.suffix; - final byte[] bytes = readAll(directory.resolve(fileName), fileName); + final byte[] bytes = readAll(directory.resolve(fileName)); final String content = new String(bytes, StandardCharsets.ISO_8859_1); int lineNumber = 0; int lineStart = 0; @@ -399,13 +403,13 @@ private static String cleanLemma(String word, String fileName, int lineNumber) */ private static int parseOffset(String offset, Tokenizer tokens) throws InvalidFormatException { if (offset.length() != 8) { - throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); + throw tokens.malformedToken(MALFORMED_OFFSET + offset); } int value = 0; for (int i = 0; i < 8; i++) { final char c = offset.charAt(i); if (c < '0' || c > '9') { - throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); + throw tokens.malformedToken(MALFORMED_OFFSET + offset); } value = value * 10 + (c - '0'); } @@ -433,13 +437,12 @@ private static char posChar(String pos, Tokenizer tokens) throws InvalidFormatEx /** * Reads a required database file in full. * - * @param file The file path. - * @param fileName The file name, for error reporting. + * @param file The file path. * @return The file bytes. * @throws InvalidFormatException Thrown if the file is missing. * @throws IOException Thrown if reading fails. */ - private static byte[] readAll(Path file, String fileName) throws IOException { + private static byte[] readAll(Path file) throws IOException { if (!Files.isRegularFile(file)) { throw new InvalidFormatException("Missing WNDB database file: " + file); } @@ -456,8 +459,19 @@ private static byte[] readAll(Path file, String fileName) throws IOException { */ private static InvalidFormatException malformed(String fileName, int lineNumber, String message) { - return new InvalidFormatException( - "Malformed WNDB file " + fileName + " at line " + lineNumber + ": " + message); + return new InvalidFormatException(malformedMessage(fileName, lineNumber, message)); + } + + /** + * Builds the malformed-file message naming the file and line. + * + * @param fileName The file name. + * @param lineNumber The 1-based line number. + * @param message The failure detail. + * @return The message text. + */ + private static String malformedMessage(String fileName, int lineNumber, String message) { + return "Malformed WNDB file " + fileName + " at line " + lineNumber + ": " + message; } /** A cursor over one line's space-separated fields. */ @@ -515,8 +529,8 @@ int nextInt(String field, int radix) throws InvalidFormatException { try { return Integer.parseInt(token, radix); } catch (NumberFormatException e) { - throw new InvalidFormatException(malformed(fileName, lineNumber, - "Field " + field + " is not a base-" + radix + " integer: " + token).getMessage(), e); + throw new InvalidFormatException(malformedMessage(fileName, lineNumber, + "Field " + field + " is not a base-" + radix + " integer: " + token), e); } } @@ -557,6 +571,7 @@ InvalidFormatException malformedToken(String message) { private record RawPointer(WordNetRelation relation, String targetId, int lineNumber) { } + /** A parsed data-file synset, kept until its pointer targets can be resolved. */ private static final class RawSynset { private final String id; private final WordNetPOS pos; diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java index 3d32eb4431..a503157ed9 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java @@ -63,4 +63,9 @@ void testLemmaKeyAndExceptionLookupAgreeOnTheFold() { void testFoldRejectsNull() { assertThrows(IllegalArgumentException.class, () -> LemmaFolding.fold(null)); } + + @Test + void testSplitOnSpacesRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> LemmaFolding.splitOnSpaces(null)); + } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java index 0292f7ffe2..a3578c2e6c 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java @@ -71,14 +71,14 @@ void testConcurrentLookupsSeeConsistentResults() throws InterruptedException { } private static void verifyOnce(LexicalKnowledgeBase lexicon, Queue problems) { - if (!"wndb-00001075-n".equals(lexicon.lookup("dog", WordNetPOS.NOUN).get(0).id())) { + if (!WndbReaderTest.DOG_ID.equals(lexicon.lookup("dog", WordNetPOS.NOUN).get(0).id())) { problems.add("Wrong dog lookup"); } if (lexicon.lookup("run", WordNetPOS.NOUN).size() != 2) { problems.add("Wrong run sense count"); } - if (!List.of("wndb-00001160-n") - .equals(lexicon.related("wndb-00001075-n", WordNetRelation.HYPERNYM))) { + if (!List.of(WndbReaderTest.CANID_ID) + .equals(lexicon.related(WndbReaderTest.DOG_ID, WordNetRelation.HYPERNYM))) { problems.add("Wrong dog hypernym"); } if (lexicon.contains("zebra", WordNetPOS.NOUN)) { diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java index b3aaf78ba2..b672b1e3c2 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java @@ -35,6 +35,11 @@ public class MorphyExceptionsTest { + /** + * Loads the exception lists from the miniature WNDB fixture directory. + * + * @return The loaded fixture exception lists. + */ static MorphyExceptions fixture() { try { return MorphyExceptions.load(WndbReaderTest.fixtureDirectory()); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java index 24df815548..e3349176dd 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java @@ -16,6 +16,7 @@ */ package opennlp.wordnet; +import java.util.Collections; import java.util.List; import org.junit.jupiter.api.Test; @@ -184,5 +185,10 @@ void testRejectsNullOrMismatchedSequences() { () -> morphy.lemmatize(new String[] {null}, new String[] {"NN"})); assertThrows(IllegalArgumentException.class, () -> morphy.lemmatize(new String[] {"dog"}, new String[] {null})); + final List withNull = Collections.singletonList(null); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(withNull, List.of("NN"))); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(List.of("dog"), withNull)); } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java index 5d48d17969..fd8c35ffa9 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java @@ -42,6 +42,11 @@ public class WnLmfReaderTest { + /** + * Loads the miniature WN-LMF document from the test classpath into a lexicon. + * + * @return The loaded fixture lexicon. + */ static LexicalKnowledgeBase fixture() { try (InputStream in = WnLmfReaderTest.class.getResourceAsStream("mini-wn-lmf.xml")) { assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath"); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java index 5931957e3b..a8eb567e47 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java @@ -28,6 +28,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import opennlp.tools.util.InvalidFormatException; import opennlp.tools.wordnet.LexicalKnowledgeBase; @@ -43,9 +45,17 @@ public class WndbReaderTest { - private static final String DOG_ID = "wndb-00001075-n"; - private static final String CANID_ID = "wndb-00001160-n"; + /** The minted id of the fixture's dog synset, shared with the other tests over this fixture. */ + static final String DOG_ID = "wndb-00001075-n"; + /** The minted id of the fixture's canid synset, the hypernym of {@link #DOG_ID}. */ + static final String CANID_ID = "wndb-00001160-n"; + + /** + * Locates the miniature WNDB database directory on the test classpath. + * + * @return The fixture directory. + */ static Path fixtureDirectory() { final URL url = WndbReaderTest.class.getResource("mini-wndb"); assertNotNull(url, "Fixture directory mini-wndb must be on the test classpath"); @@ -56,6 +66,11 @@ static Path fixtureDirectory() { } } + /** + * Loads the miniature WNDB database into a lexicon. + * + * @return The loaded fixture lexicon. + */ static LexicalKnowledgeBase fixture() { try { return WndbReader.read(fixtureDirectory()); @@ -243,6 +258,28 @@ void testDanglingPointerErrorNamesPointerLine(@TempDir Path tempDir) throws IOEx assertTrue(e.getMessage().contains("line 1")); } + @ParameterizedTest + @CsvSource({ + // One field-level rejection per row, each driven by a same-length edit of a single fixture + // line so that every following line's byte offset stays valid. + "data.noun, 00001564 03 n, 0000156x 03 n, Synset offset must be 8 digits", + "data.noun, 00001075 03 n, 00001075 03 v, Synset type v does not belong in", + "data.noun, n 01 box, n 0z box, is not a base-16 integer: 0z", + "data.noun, n 01 man, n 00 man, Word count must be at least 1", + "data.noun, 00001160 n 0000, 00001160 q 0000, Pointer pos must be one of", + "data.noun, | a domesticated canid, ! a domesticated canid, Expected the | gloss separator", + "data.adj, short(p), short(x), Unknown syntactic marker on word: short(x)", + "index.noun, berry n 1, berry v 1, Index pos v does not belong in", + }) + void testRejectsMalformedField(String fileName, String find, String replacement, + String expected, @TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, fileName, line -> line.replace(find, replacement)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains(expected), e.getMessage()); + } + private static void writeEmptyDb(Path directory, String... suffixes) throws IOException { for (final String suffix : suffixes) { Files.writeString(directory.resolve("data." + suffix), ""); @@ -258,8 +295,9 @@ private static void copyFixture(Path target) throws IOException { } } - // Applies a line transformation to one fixture file. The mutations only ever keep or shrink - // line lengths of the affected line's own fields, so surrounding offsets stay valid. + // Applies a line transformation to one fixture file. A same-length edit keeps every following + // line's byte offset valid; an edit that changes a line's length is only safe when the reader + // is expected to fail on that line itself, before it reads the ones after it. private static void mutate(Path directory, String fileName, UnaryOperator edit) throws IOException { final Path file = directory.resolve(fileName); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java index 530e25ce66..9f4b46aff5 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java @@ -14,23 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package opennlp.wordnet; import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Path; import java.util.List; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; import opennlp.tools.wordnet.WordNetPOS; +import static org.junit.jupiter.api.Assertions.assertEquals; + /** * Runs the manual's WordNet examples (docbkx {@code wordnet.xml}) verbatim: every value * the chapter states is asserted here, so a change breaking this test breaks the manual. @@ -39,40 +35,23 @@ */ public class WordNetUsageExampleTest { - private static LexicalKnowledgeBase loadMiniWnLmf() throws IOException { - try (InputStream in = WordNetUsageExampleTest.class.getResourceAsStream("mini-wn-lmf.xml")) { - Assertions.assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath"); - return WnLmfReader.read(in, "mini-wn-lmf.xml"); - } - } - - private static Path miniWndbDirectory() { - final URL url = WordNetUsageExampleTest.class.getResource("mini-wndb"); - Assertions.assertNotNull(url, "Fixture directory mini-wndb must be on the test classpath"); - try { - return Path.of(url.toURI()); - } catch (URISyntaxException e) { - throw new IllegalStateException("Unexpected fixture URI: " + url, e); - } - } - /** * Load, lookup, and Morphy lemmatize as the chapter shows. */ @Test void testLoadLookupAndLemmatize() throws IOException { - final LexicalKnowledgeBase lexicon = loadMiniWnLmf(); + final LexicalKnowledgeBase lexicon = WnLmfReaderTest.fixture(); final List senses = lexicon.lookup("dog", WordNetPOS.NOUN); - Assertions.assertEquals(1, senses.size()); - Assertions.assertEquals("mini-n1", senses.get(0).id()); - Assertions.assertEquals(List.of("dog", "domestic dog"), senses.get(0).lemmas()); - Assertions.assertEquals("a domesticated canid", senses.get(0).gloss()); - - final MorphyLemmatizer lemmatizer = - new MorphyLemmatizer(lexicon, MorphyExceptions.load(miniWndbDirectory())); - Assertions.assertEquals("mouse", + assertEquals(1, senses.size()); + assertEquals("mini-n1", senses.get(0).id()); + assertEquals(List.of("dog", "domestic dog"), senses.get(0).lemmas()); + assertEquals("a domesticated canid", senses.get(0).gloss()); + + final MorphyLemmatizer lemmatizer = new MorphyLemmatizer(lexicon, + MorphyExceptions.load(WndbReaderTest.fixtureDirectory())); + assertEquals("mouse", lemmatizer.lemmatize(new String[] {"mice"}, new String[] {"NNS"})[0]); - Assertions.assertEquals("dog", + assertEquals("dog", lemmatizer.lemmatize(new String[] {"dogs"}, new String[] {"NNS"})[0]); } } From c678d6579c6c6496a5570a88aae4479753c69fb7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:31:54 -0400 Subject: [PATCH 05/15] OPENNLP-1880: Document relation navigation and close mirror-test and reader-test gaps Add a Navigating relations section to the WordNet chapter and mirror it, pin the chapter's Path-based loading listing with a usage-test method, give the exception fixture a two-base-form entry (axes to axis, ax) so the list-form lemmatizer path is exercised with multiple candidates, and fold the copy-paste WN-LMF rejection tests into one parameterized ladder with identical documents and message pins. --- opennlp-docs/src/docbkx/wordnet.xml | 24 ++ .../opennlp/wordnet/MorphyLemmatizerTest.java | 11 +- .../java/opennlp/wordnet/WnLmfReaderTest.java | 247 ++++++++---------- .../wordnet/WordNetUsageExampleTest.java | 44 ++++ .../opennlp/wordnet/mini-wndb/noun.exc | 1 + 5 files changed, 191 insertions(+), 136 deletions(-) diff --git a/opennlp-docs/src/docbkx/wordnet.xml b/opennlp-docs/src/docbkx/wordnet.xml index 7b01e99860..e3db734d15 100644 --- a/opennlp-docs/src/docbkx/wordnet.xml +++ b/opennlp-docs/src/docbkx/wordnet.xml @@ -61,6 +61,30 @@ List senses = lexicon.lookup("dog", WordNetPOS.NOUN); +
+ Navigating relations + + Each synset carries its typed relations (WordNetRelation) to + other synsets. Follow a relation with Synset.related(...), + resolve the returned ids with synset(id), or navigate in one + step with related(...) on the knowledge base. Starting from + the noun dog above, the hypernym relation leads to its + parent concept: + parents = dog.related(WordNetRelation.HYPERNYM); +// parents = ["mini-n2"] + +Synset parent = lexicon.synset(parents.get(0)).orElseThrow(); +// parent.lemmas() = ["canid"] +// parent.gloss() = "a carnivorous mammal with nonretractile claws" + +// one-step navigation, back down the hyponym relation: +// lexicon.related("mini-n2", WordNetRelation.HYPONYM) = ["mini-n1"]]]> + + +
+
Morphy lemmatization diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java index e3349176dd..7d2b5f4dbc 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java @@ -117,9 +117,18 @@ void testArrayFormKeepsPositions() { @Test void testListFormReturnsAllCandidates() { final List> lemmas = morphy().lemmatize( - List.of("glarbs", "berries"), List.of("NNS", "NNS")); + List.of("glarbs", "berries", "axes"), List.of("NNS", "NNS", "NNS")); assertEquals(List.of("O"), lemmas.get(0)); assertEquals(List.of("berry"), lemmas.get(1)); + // The fixture noun.exc lists axes with two base forms; both come back, in file order. + assertEquals(List.of("axis", "ax"), lemmas.get(2)); + } + + @Test + void testArrayFormReturnsFirstOfSeveralCandidates() { + // The list form above returns both base forms of axes; the array form keeps only the + // first, most preferred one. + assertEquals("axis", one("axes", "NNS")); } @Test diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java index fd8c35ffa9..0370a61f76 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java @@ -23,9 +23,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import opennlp.tools.util.InvalidFormatException; import opennlp.tools.wordnet.LexicalKnowledgeBase; @@ -248,40 +253,113 @@ void testRejectsTruncatedDocument() { assertTrue(e.getMessage().contains("inline.xml")); } - @Test - void testRejectsSenseWithoutSynsetAttribute() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + ""))); - assertTrue(e.getMessage().contains("synset")); - } - - @Test - void testRejectsSenseToUndeclaredSynset() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + ""))); - assertTrue(e.getMessage().contains("t-9")); - } - - @Test - void testRejectsRelationToUndeclaredSynset() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "a feline" - + ""))); - assertTrue(e.getMessage().contains("t-9")); - } - - @Test - void testRejectsUnknownRelationType() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "a feline" - + ""))); - assertTrue(e.getMessage().contains("quasi_synonym")); + /** + * One rejected document per structural rule the reader enforces: the document body (wrapped + * in the standard resource envelope) and the fragments its rejection message must contain. + * + * @return The (description, document body, expected message fragments) cases. + */ + static Stream rejectedDocuments() { + return Stream.of( + Arguments.of(Named.of("sense without synset attribute", + "" + + ""), + List.of("synset")), + Arguments.of(Named.of("sense to undeclared synset", + "" + + ""), + List.of("t-9")), + Arguments.of(Named.of("relation to undeclared synset", + "" + + "" + + "a feline" + + ""), + List.of("t-9")), + Arguments.of(Named.of("unknown relation type", + "" + + "" + + "a feline" + + ""), + List.of("quasi_synonym")), + Arguments.of(Named.of("unknown part of speech", + "" + + "" + + "a feline" + + ""), + List.of("x")), + Arguments.of(Named.of("synset without members", + "orphan"), + List.of("t-1")), + Arguments.of(Named.of("duplicate synset id", + "" + + "" + + "a feline" + + "" + + "a repeat" + + ""), + List.of("Duplicate synset id t-1")), + Arguments.of(Named.of("duplicate lexical entry id", + "" + + "" + + "" + + "" + + "a feline" + + ""), + List.of("Duplicate lexical entry id t-cat-n")), + Arguments.of(Named.of("duplicate sense id", + "" + + "" + + "" + + "a feline" + + "" + + "a second" + + ""), + List.of("Duplicate sense id t-cat-n-1")), + Arguments.of(Named.of("synset member pos mismatch", + "" + + "" + + "a feline" + + ""), + List.of("t-cat-n", "VERB", "NOUN")), + Arguments.of(Named.of("sense relation to undeclared sense", + "" + + "" + + "" + + "" + + "a feline" + + ""), + List.of("t-ghost-1")), + Arguments.of(Named.of("lemma outside lexical entry", + ""), + List.of("Lemma outside a LexicalEntry")), + Arguments.of(Named.of("sense before lemma", + "" + + "" + + "a feline" + + ""), + List.of("Sense before its entry's Lemma")), + Arguments.of(Named.of("sense relation outside sense", + "" + + "" + + "" + + "a feline" + + ""), + List.of("SenseRelation outside a Sense")), + Arguments.of(Named.of("synset relation outside synset", + ""), + List.of("SynsetRelation outside a Synset"))); + } + + @ParameterizedTest + @MethodSource("rejectedDocuments") + void testRejectsStructurallyInvalidDocument(String body, + List expectedMessageFragments) { + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> parse(wrap(body))); + for (final String fragment : expectedMessageFragments) { + assertTrue(e.getMessage().contains(fragment), + () -> "Rejection message must contain '" + fragment + "' but was: " + e.getMessage()); + } } @Test @@ -306,105 +384,4 @@ void testSkipsOtherRelationTypeOnSynsetRelation() throws IOException { assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty()); } - @Test - void testRejectsUnknownPartOfSpeech() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "a feline"))); - assertTrue(e.getMessage().contains("x")); - } - - @Test - void testRejectsSynsetWithoutMembers() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("orphan"))); - assertTrue(e.getMessage().contains("t-1")); - } - - @Test - void testRejectsDuplicateSynsetId() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "a feline" - + "a repeat"))); - assertTrue(e.getMessage().contains("Duplicate synset id t-1")); - } - - @Test - void testRejectsDuplicateLexicalEntryId() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "" - + "" - + "a feline"))); - assertTrue(e.getMessage().contains("Duplicate lexical entry id t-cat-n")); - } - - @Test - void testRejectsDuplicateSenseId() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "" - + "a feline" - + "a second"))); - assertTrue(e.getMessage().contains("Duplicate sense id t-cat-n-1")); - } - - @Test - void testRejectsSynsetMemberPosMismatch() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "a feline"))); - assertTrue(e.getMessage().contains("t-cat-n")); - assertTrue(e.getMessage().contains("VERB")); - assertTrue(e.getMessage().contains("NOUN")); - } - - @Test - void testRejectsSenseRelationToUndeclaredSense() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "" - + "a feline"))); - assertTrue(e.getMessage().contains("t-ghost-1")); - } - - @Test - void testRejectsLemmaOutsideLexicalEntry() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, - () -> parse(wrap(""))); - assertTrue(e.getMessage().contains("Lemma outside a LexicalEntry")); - } - - @Test - void testRejectsSenseBeforeLemma() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "a feline"))); - assertTrue(e.getMessage().contains("Sense before its entry's Lemma")); - } - - @Test - void testRejectsSenseRelationOutsideSense() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( - wrap("" - + "" - + "" - + "a feline"))); - assertTrue(e.getMessage().contains("SenseRelation outside a Sense")); - } - - @Test - void testRejectsSynsetRelationOutsideSynset() { - final InvalidFormatException e = assertThrows(InvalidFormatException.class, - () -> parse(wrap(""))); - assertTrue(e.getMessage().contains("SynsetRelation outside a Synset")); - } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java index 9f4b46aff5..0288d66332 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WordNetUsageExampleTest.java @@ -17,15 +17,21 @@ package opennlp.wordnet; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Runs the manual's WordNet examples (docbkx {@code wordnet.xml}) verbatim: every value @@ -54,4 +60,42 @@ void testLoadLookupAndLemmatize() throws IOException { assertEquals("dog", lemmatizer.lemmatize(new String[] {"dogs"}, new String[] {"NNS"})[0]); } + + /** + * Load through the Path entry points exactly as the chapter's loading listing shows: + * {@code WnLmfReader.read(Path)} on a file named {@code en-wordnet.xml} (a temp-dir copy of + * the fixture) and {@code WndbReader.read(Path)} on a WNDB {@code dict} directory. + */ + @Test + void testLoadFromPath(@TempDir Path tempDir) throws IOException { + final Path file = tempDir.resolve("en-wordnet.xml"); + try (InputStream in = WnLmfReaderTest.class.getResourceAsStream("mini-wn-lmf.xml")) { + assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath"); + Files.copy(in, file); + } + final LexicalKnowledgeBase lexicon = WnLmfReader.read(file); + assertEquals("mini-n1", lexicon.lookup("dog", WordNetPOS.NOUN).get(0).id()); + + final LexicalKnowledgeBase wndbLexicon = WndbReader.read(WndbReaderTest.fixtureDirectory()); + assertEquals(List.of("dog", "domestic dog"), + wndbLexicon.lookup("dog", WordNetPOS.NOUN).get(0).lemmas()); + } + + /** + * Follow the hypernym relation from the first sense of dog as the chapter's relation + * navigation listing shows. + */ + @Test + void testNavigateRelations() { + final LexicalKnowledgeBase lexicon = WnLmfReaderTest.fixture(); + final Synset dog = lexicon.lookup("dog", WordNetPOS.NOUN).get(0); + final List parents = dog.related(WordNetRelation.HYPERNYM); + assertEquals(List.of("mini-n2"), parents); + + final Synset parent = lexicon.synset(parents.get(0)).orElseThrow(); + assertEquals(List.of("canid"), parent.lemmas()); + assertEquals("a carnivorous mammal with nonretractile claws", parent.gloss()); + + assertEquals(List.of("mini-n1"), lexicon.related("mini-n2", WordNetRelation.HYPONYM)); + } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc index e5b3080a88..71b2d55fb6 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc @@ -1,3 +1,4 @@ +axes axis ax men man mice mouse oxen ox From 9c4d2f9f6e0fe8072dfc8853904d4b7cba959245 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 07:14:51 -0400 Subject: [PATCH 06/15] OPENNLP-1888: Add StringUtil.isBlank following the toolkit whitespace definition A blank check under the toolkit's whitespace definition, which unlike String.isBlank covers the no-break spaces, so annotators validating labels and identifiers share one predicate instead of each carrying a private copy. Reads whole code points; tests pin the no-break and figure spaces, the empty string, and a supplementary-plane letter. --- .../java/opennlp/tools/util/StringUtil.java | 21 +++++++++++++++++++ .../opennlp/tools/util/StringUtilTest.java | 19 +++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java index 98cca59891..1842556720 100644 --- a/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java +++ b/opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java @@ -268,6 +268,27 @@ public static boolean isEmpty(CharSequence theString) { return theString.length() == 0; } + /** + * Determines whether a {@link CharSequence} is blank: empty, or made up entirely of + * code points that {@link #isWhitespace(int)} accepts. Unlike + * {@link String#isBlank()}, this follows the toolkit's whitespace definition, which + * includes the no-break spaces the JDK predicate leaves out, so a value spelled + * entirely from them cannot pass a blank check as content. + * + * @param theString The {@link CharSequence} to examine. Must not be {@code null}. + * @return {@code true} if {@code theString} is empty or all whitespace. + */ + public static boolean isBlank(CharSequence theString) { + for (int i = 0; i < theString.length(); ) { + final int codePoint = Character.codePointAt(theString, i); + if (!isWhitespace(codePoint)) { + return false; + } + i += Character.charCount(codePoint); + } + return true; + } + /** * Get the minimum of three values. * diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java index a47306d3d4..73f81c209e 100644 --- a/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java +++ b/opennlp-tools/src/test/java/opennlp/tools/util/StringUtilTest.java @@ -679,4 +679,23 @@ void testLowercaseBeyondBMP() { String lc = StringUtil.toLowerCase(input); Assertions.assertArrayEquals(expectedCodePoints, lc.codePoints().toArray()); } + + /** + * Verifies the blank check against the toolkit's whitespace definition: the + * no-break space is blank here although the JDK's own check does not cover it, + * whitespace-only and empty values are blank, and any non-whitespace code point, + * supplementary ones included, makes a value non-blank. + */ + @Test + void testIsBlankFollowsTheToolkitWhitespaceDefinition() { + Assertions.assertTrue(StringUtil.isBlank("")); + Assertions.assertTrue(StringUtil.isBlank(" \t\n")); + // U+00A0 no-break space and U+2007 figure space: JDK String.isBlank says false + Assertions.assertTrue(StringUtil.isBlank("\u00A0")); + Assertions.assertTrue(StringUtil.isBlank(" \u00A0\u2007 ")); + Assertions.assertFalse(StringUtil.isBlank("a")); + Assertions.assertFalse(StringUtil.isBlank(" a ")); + // U+10428, a supplementary-plane letter read as one code point, not two chars + Assertions.assertFalse(StringUtil.isBlank("\uD801\uDC28")); + } } From 22276921bd9ad9b901d8effb1fab24801d88f8c6 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 16:45:20 -0400 Subject: [PATCH 07/15] OPENNLP-1887: Add LexicalExpander: weighted synonym and hypernym expansion over the lexical knowledge base Expands a term into the synonyms sharing its synsets, the lemmas of its hypernym ancestors up to a configured depth (following both the direct and the instance relation), and optionally its direct hyponyms. Each expansion carries a deterministic heuristic weight: sense rank and every relation step multiply configurable decays, so consumers can discount looser expansions instead of treating them as the original term. Results exclude the input, deduplicate case-insensitively keeping the highest weight, and order stably by weight, kind, and term. Inflected input works through an optional Lemmatizer invoked with the WordNetPos name as the tag, which the Morphy lemmatizer understands: dogs expands through dog, mice through the exception list to mouse. Hypernym walks are visited-checked so malformed cyclic data terminates. Tests cover the graph behavior on a hand-built lexicon (sense ranking, decay arithmetic, dedup, cycle termination, validation) and run the whole stack over the miniature WN-LMF and WNDB fixtures, asserting both readers produce identical expansions. --- .../java/opennlp/wordnet/LexicalExpander.java | 487 ++++++++++++++++++ .../opennlp/wordnet/ExpansionAssertions.java | 42 ++ .../wordnet/LexicalExpanderLexiconTest.java | 106 ++++ .../opennlp/wordnet/LexicalExpanderTest.java | 331 ++++++++++++ 4 files changed, 966 insertions(+) create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ExpansionAssertions.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java new file mode 100644 index 0000000000..b21f45126f --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.lemmatizer.Lemmatizer; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Expands a term into related terms drawn from a {@link LexicalKnowledgeBase}: the synonyms + * sharing its synsets, the lemmas of its hypernym ancestors up to a configured depth, and + * optionally the lemmas of its direct hyponyms. + * + *

Each {@link Expansion} carries a deterministic heuristic weight, not a probability: the + * first sense of a term starts at {@code 1.0}, each later sense is multiplied by the configurable + * sense decay, and every hypernym or hyponym step multiplies by the configurable depth decay. + * A decay product that underflows to zero in double arithmetic carries no ranking signal, so + * such expansions are dropped rather than emitted outside the {@code (0, 1]} weight range. + * When the term itself is not in the lexicon and a {@link Lemmatizer} is configured, the term is + * lemmatized and the lemma expanded instead; the lemmatizer is invoked with the + * {@link WordNetPOS} name as the tag.

+ * + *

Hypernym walks follow both the direct and the instance relation, track visited synsets so + * malformed cyclic data cannot loop, and never report the term itself. Results are deduplicated + * case-insensitively, keeping the highest weight, and ordered by weight descending, then kind, + * then term, so output is stable across runs.

+ * + *

Instances are immutable and safe for concurrent use when the configured lexicon and + * lemmatizer are.

+ */ +@ThreadSafe +public final class LexicalExpander { + + /** How an expansion relates to the input term. */ + public enum Kind { + + /** A member of one of the term's own synsets. */ + SYNONYM, + + /** A lemma of an ancestor synset, {@link Expansion#depth()} steps up. */ + HYPERNYM, + + /** A lemma of a direct child synset. */ + HYPONYM + } + + /** + * One expansion of a term. + * + * @param term The expanded term, in the lexicon's written form (multiword terms contain + * spaces). + * @param kind How the term relates to the input. + * @param depth The relation distance: {@code 0} for synonyms, the number of hypernym steps + * for {@link Kind#HYPERNYM}, {@code 1} for hyponyms. + * @param senseRank The zero-based rank of the input sense this expansion came from. + * @param weight The heuristic weight in {@code (0, 1]}; higher is closer to the input term. + */ + public record Expansion(String term, Kind kind, int depth, int senseRank, double weight) { + + /** + * Validates every component against the documented ranges. + * + * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or blank, + * {@code kind} is {@code null}, {@code depth} or {@code senseRank} is negative, or + * {@code weight} is not in {@code (0, 1]}. + */ + public Expansion { + if (term == null || term.isBlank()) { + throw new IllegalArgumentException("term must not be null or blank"); + } + if (kind == null) { + throw new IllegalArgumentException("kind must not be null"); + } + if (depth < 0) { + throw new IllegalArgumentException("depth must not be negative: " + depth); + } + if (senseRank < 0) { + throw new IllegalArgumentException("senseRank must not be negative: " + senseRank); + } + if (!(weight > 0.0 && weight <= 1.0)) { + throw new IllegalArgumentException("weight must be in (0, 1]: " + weight); + } + } + } + + private final LexicalKnowledgeBase lexicon; + private final Lemmatizer lemmatizer; + private final int maxSenses; + private final int hypernymDepth; + private final boolean includeHyponyms; + private final int maxExpansions; + private final double senseDecay; + private final double depthDecay; + + /** + * Creates an expander from a builder whose fields have already been validated. + * + * @param builder The configured builder. Must not be {@code null}. + */ + private LexicalExpander(Builder builder) { + this.lexicon = builder.lexicon; + this.lemmatizer = builder.lemmatizer; + this.maxSenses = builder.maxSenses; + this.hypernymDepth = builder.hypernymDepth; + this.includeHyponyms = builder.includeHyponyms; + this.maxExpansions = builder.maxExpansions; + this.senseDecay = builder.senseDecay; + this.depthDecay = builder.depthDecay; + } + + /** + * Starts a builder. + * + * @param lexicon The knowledge base to expand against; must not be null. + * @return A builder with the default configuration. + * @throws IllegalArgumentException Thrown if {@code lexicon} is null. + */ + public static Builder builder(LexicalKnowledgeBase lexicon) { + return new Builder(lexicon); + } + + /** + * Expands a term for one part of speech. + * + * @param term The term to expand; must not be null or blank. + * @param pos The part of speech to expand as; must not be null. + * @return The expansions, deduplicated and ordered by descending weight; empty when the term + * (and its lemma, when a lemmatizer is configured) is not in the lexicon. + * @throws IllegalArgumentException Thrown if {@code term} is null or blank or {@code pos} is + * null. + */ + public List expand(String term, WordNetPOS pos) { + if (pos == null) { + throw new IllegalArgumentException("The pos must not be null."); + } + return collect(term, List.of(pos)); + } + + /** + * Expands a term across all parts of speech. + * + * @param term The term to expand; must not be null or blank. + * @return The expansions across every part of speech, deduplicated and ordered by descending + * weight; empty when the term is not in the lexicon. + * @throws IllegalArgumentException Thrown if {@code term} is null or blank. + */ + public List expand(String term) { + return collect(term, List.of(WordNetPOS.values())); + } + + /** + * Expands the term across the given parts of speech and returns the ranked, capped result. + * + * @param term The term to expand. Must not be {@code null} or blank. + * @param poses The parts of speech to expand as. Must not be {@code null}. + * @return The expansions, deduplicated, ordered by descending weight, and capped at the + * configured maximum; empty when the term is not in the lexicon. + * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or blank. + */ + private List collect(String term, List poses) { + if (term == null || term.isBlank()) { + throw new IllegalArgumentException("The term must not be null or blank."); + } + final Map best = new HashMap<>(); + final Set excluded = new HashSet<>(); + excluded.add(LemmaFolding.fold(term)); + + for (final WordNetPOS pos : poses) { + final String subject = resolveSubject(term, pos); + if (subject == null) { + continue; + } + final List senses = lexicon.lookup(subject, pos); + final int senseCount = Math.min(senses.size(), maxSenses); + for (int rank = 0; rank < senseCount; rank++) { + final double senseWeight = Math.pow(senseDecay, rank); + expandSense(senses.get(rank), rank, senseWeight, best, excluded); + } + } + + final List ordered = new ArrayList<>(best.values()); + ordered.sort(Comparator.comparingDouble(Expansion::weight).reversed() + .thenComparing(Expansion::kind) + .thenComparing(Expansion::term)); + return ordered.size() > maxExpansions ? List.copyOf(ordered.subList(0, maxExpansions)) + : List.copyOf(ordered); + } + + /** + * Resolves the form actually expanded: the term when the lexicon knows it, otherwise its lemma + * when a lemmatizer is configured and produces a known lemma. + * + * @param term The input term. Must not be {@code null}. + * @param pos The part of speech to look the term up as. Must not be {@code null}. + * @return The known form to expand, or {@code null} when neither the term nor its lemma is in + * the lexicon. + */ + private String resolveSubject(String term, WordNetPOS pos) { + if (lexicon.contains(term, pos)) { + return term; + } + if (lemmatizer == null) { + return null; + } + final String[] lemmas = + lemmatizer.lemmatize(new String[] {term}, new String[] {pos.name()}); + if (lemmas.length == 0 || lemmas[0] == null) { + return null; + } + final String lemma = lemmas[0]; + // Lemmatizers report an unresolvable token with the contract's unknown marker. + if (MorphyLemmatizer.UNKNOWN_LEMMA.equals(lemma) || !lexicon.contains(lemma, pos)) { + return null; + } + return lemma; + } + + /** + * Offers the synonyms, hypernym ancestors, and optional hyponyms of one sense into the running + * best-expansion map. + * + * @param sense The sense to expand. Must not be {@code null}. + * @param rank The zero-based salience rank of the sense. + * @param senseWeight The weight of the sense itself; each relation step decays from it. + * @param best The best expansion seen so far per folded term; updated in place. + * @param excluded The folded terms that are never reported, such as the input term. + */ + private void expandSense(Synset sense, int rank, double senseWeight, + Map best, Set excluded) { + if (senseWeight == 0.0) { + // The decay product underflowed to zero in double arithmetic. A zero weight + // carries no ranking signal, so the sense and everything derived from it is + // dropped instead of emitted outside the documented (0, 1] weight range. + return; + } + for (final String lemma : sense.lemmas()) { + offer(best, excluded, new Expansion(lemma, Kind.SYNONYM, 0, rank, senseWeight)); + } + + // Breadth-first hypernym walk, visited-checked so cyclic data terminates. + if (hypernymDepth > 0) { + final Set visited = new HashSet<>(); + visited.add(sense.id()); + final ArrayDeque frontier = new ArrayDeque<>(hypernymsOf(sense)); + final ArrayDeque next = new ArrayDeque<>(); + for (int depth = 1; depth <= hypernymDepth && !frontier.isEmpty(); depth++) { + final double depthWeight = senseWeight * Math.pow(depthDecay, depth); + if (depthWeight == 0.0) { + // Deeper levels only shrink further, so the walk stops at the first underflow. + break; + } + while (!frontier.isEmpty()) { + final String id = frontier.poll(); + if (!visited.add(id)) { + continue; + } + final Synset ancestor = lexicon.synset(id).orElse(null); + if (ancestor == null) { + continue; + } + for (final String lemma : ancestor.lemmas()) { + offer(best, excluded, + new Expansion(lemma, Kind.HYPERNYM, depth, rank, depthWeight)); + } + next.addAll(hypernymsOf(ancestor)); + } + frontier.addAll(next); + next.clear(); + } + } + + if (includeHyponyms && senseWeight * depthDecay > 0.0) { + final double hyponymWeight = senseWeight * depthDecay; + final List children = new ArrayList<>(sense.related(WordNetRelation.HYPONYM)); + children.addAll(sense.related(WordNetRelation.INSTANCE_HYPONYM)); + for (final String id : children) { + lexicon.synset(id).ifPresent(child -> { + for (final String lemma : child.lemmas()) { + offer(best, excluded, new Expansion(lemma, Kind.HYPONYM, 1, rank, hyponymWeight)); + } + }); + } + } + } + + /** + * Collects the synset ids of both the direct and the instance hypernyms of a synset. + * + * @param synset The synset whose hypernyms are collected. Must not be {@code null}. + * @return The hypernym synset ids in source order, direct relations first. + */ + private static List hypernymsOf(Synset synset) { + final List direct = synset.related(WordNetRelation.HYPERNYM); + final List instance = synset.related(WordNetRelation.INSTANCE_HYPERNYM); + if (instance.isEmpty()) { + return direct; + } + final List all = new ArrayList<>(direct.size() + instance.size()); + all.addAll(direct); + all.addAll(instance); + return all; + } + + /** + * Records the candidate under its folded term when it is not excluded and it beats the current + * best weight for that term. Folding through {@link LemmaFolding#fold(String)} keeps the + * exclusion and deduplication keys aligned with the lexicon's own lemma keys. + * + * @param best The best expansion seen so far per folded term; updated in place. + * @param excluded The folded terms that are never reported. + * @param candidate The expansion to offer. Must not be {@code null}. + */ + private static void offer(Map best, Set excluded, + Expansion candidate) { + final String key = LemmaFolding.fold(candidate.term()); + if (excluded.contains(key)) { + return; + } + final Expansion current = best.get(key); + if (current == null || candidate.weight() > current.weight()) { + best.put(key, candidate); + } + } + + /** Configures and creates a {@link LexicalExpander}. */ + public static final class Builder { + + private final LexicalKnowledgeBase lexicon; + private Lemmatizer lemmatizer; + private int maxSenses = 3; + private int hypernymDepth = 1; + private boolean includeHyponyms = false; + private int maxExpansions = 20; + private double senseDecay = 0.5; + private double depthDecay = 0.5; + + /** + * Creates a builder over the given lexicon; use {@link LexicalExpander#builder}. + * + * @param lexicon The knowledge base to expand against. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code lexicon} is null. + */ + private Builder(LexicalKnowledgeBase lexicon) { + if (lexicon == null) { + throw new IllegalArgumentException("The lexicon must not be null."); + } + this.lexicon = lexicon; + } + + /** + * Configures a lemmatizer used when the input term itself is not in the lexicon. It is + * invoked with the {@link WordNetPOS} name as the tag. + * + * @param lemmatizer The fallback lemmatizer; must not be null. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code lemmatizer} is null. + */ + public Builder lemmatizer(Lemmatizer lemmatizer) { + if (lemmatizer == null) { + throw new IllegalArgumentException("The lemmatizer must not be null."); + } + this.lemmatizer = lemmatizer; + return this; + } + + /** + * Configures how many senses of the term are expanded, most salient first. + * + * @param maxSenses The sense count; must be positive. The default is {@code 3}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code maxSenses} is not positive. + */ + public Builder maxSenses(int maxSenses) { + if (maxSenses < 1) { + throw new IllegalArgumentException("The maxSenses must be positive: " + maxSenses); + } + this.maxSenses = maxSenses; + return this; + } + + /** + * Configures how many hypernym steps are walked; {@code 0} disables hypernym expansion. + * + * @param hypernymDepth The depth; must not be negative. The default is {@code 1}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code hypernymDepth} is negative. + */ + public Builder hypernymDepth(int hypernymDepth) { + if (hypernymDepth < 0) { + throw new IllegalArgumentException( + "The hypernymDepth must not be negative: " + hypernymDepth); + } + this.hypernymDepth = hypernymDepth; + return this; + } + + /** + * Configures whether direct hyponyms are included; off by default. + * + * @param includeHyponyms Whether to include direct hyponyms. + * @return This builder. + */ + public Builder includeHyponyms(boolean includeHyponyms) { + this.includeHyponyms = includeHyponyms; + return this; + } + + /** + * Configures the maximum number of expansions returned after ranking. + * + * @param maxExpansions The cap; must be positive. The default is {@code 20}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code maxExpansions} is not positive. + */ + public Builder maxExpansions(int maxExpansions) { + if (maxExpansions < 1) { + throw new IllegalArgumentException( + "The maxExpansions must be positive: " + maxExpansions); + } + this.maxExpansions = maxExpansions; + return this; + } + + /** + * Configures the weight multiplier applied per sense rank step. + * + * @param senseDecay The decay in {@code (0, 1]}. The default is {@code 0.5}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code senseDecay} is outside {@code (0, 1]}. + */ + public Builder senseDecay(double senseDecay) { + if (!(senseDecay > 0 && senseDecay <= 1)) { + throw new IllegalArgumentException( + "The senseDecay must be in (0, 1]: " + senseDecay); + } + this.senseDecay = senseDecay; + return this; + } + + /** + * Configures the weight multiplier applied per hypernym or hyponym step. + * + * @param depthDecay The decay in {@code (0, 1]}. The default is {@code 0.5}. + * @return This builder. + * @throws IllegalArgumentException Thrown if {@code depthDecay} is outside {@code (0, 1]}. + */ + public Builder depthDecay(double depthDecay) { + if (!(depthDecay > 0 && depthDecay <= 1)) { + throw new IllegalArgumentException( + "The depthDecay must be in (0, 1]: " + depthDecay); + } + this.depthDecay = depthDecay; + return this; + } + + /** {@return the configured expander} */ + public LexicalExpander build() { + return new LexicalExpander(this); + } + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ExpansionAssertions.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ExpansionAssertions.java new file mode 100644 index 0000000000..c67d7c425e --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ExpansionAssertions.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import opennlp.wordnet.LexicalExpander.Expansion; + +/** + * Shared lookup helpers for tests that assert on {@link LexicalExpander} output. + */ +final class ExpansionAssertions { + + /** Not instantiable. */ + private ExpansionAssertions() { + } + + /** + * Finds the first expansion of a term in an expansion list. + * + * @param expansions The expansions to search. Must not be {@code null}. + * @param term The exact term to find. Must not be {@code null}. + * @return The first expansion whose term equals {@code term}, or {@code null} when absent. + */ + static Expansion find(List expansions, String term) { + return expansions.stream().filter(e -> e.term().equals(term)).findFirst().orElse(null); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java new file mode 100644 index 0000000000..c52b01da7d --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.wordnet.LexicalExpander.Expansion; +import opennlp.wordnet.LexicalExpander.Kind; + +import static opennlp.wordnet.ExpansionAssertions.find; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * End-to-end expansion over the miniature lexicon fixtures: the WN-LMF and WNDB readers each + * feed the expander, and the Morphy lemmatizer bridges inflected input, exercising the whole + * stack the way a consumer wires it. + */ +class LexicalExpanderLexiconTest { + + @Test + void testExpansionOverTheWnLmfLexicon() { + final LexicalExpander expander = + LexicalExpander.builder(WnLmfReaderTest.fixture()).build(); + + final List expansions = expander.expand("dog", WordNetPOS.NOUN); + final Expansion domesticDog = find(expansions, "domestic dog"); + assertEquals(Kind.SYNONYM, domesticDog.kind()); + assertEquals(1.0, domesticDog.weight()); + final Expansion canid = find(expansions, "canid"); + assertEquals(Kind.HYPERNYM, canid.kind()); + assertEquals(0.5, canid.weight()); + } + + @Test + void testExpansionOverTheWndbLexicon() { + final LexicalExpander expander = + LexicalExpander.builder(WndbReaderTest.fixture()).build(); + + final List expansions = expander.expand("dog", WordNetPOS.NOUN); + assertNotNull(find(expansions, "domestic dog"), "got " + expansions); + final Expansion canid = find(expansions, "canid"); + assertEquals(Kind.HYPERNYM, canid.kind()); + } + + @Test + void testUnderscoreQueryFoldsToTheMultiwordLexiconEntry() { + // The WNDB index stores the entry as "domestic_dog"; the reader folds it to "domestic dog" + // at load time. The expander must fold the underscore query the same way, so the entry's + // own synset expands and the query never surfaces as its own synonym. + final List expansions = LexicalExpander.builder(WndbReaderTest.fixture()) + .build().expand("domestic_dog", WordNetPOS.NOUN); + + assertEquals(List.of( + new Expansion("dog", Kind.SYNONYM, 0, 0, 1.0), + new Expansion("canid", Kind.HYPERNYM, 1, 0, 0.5)), expansions); + } + + @Test + void testReadersAgreeOnExpansions() { + final List lmf = LexicalExpander.builder(WnLmfReaderTest.fixture()) + .hypernymDepth(2).build().expand("mouse", WordNetPOS.NOUN); + final List wndb = LexicalExpander.builder(WndbReaderTest.fixture()) + .hypernymDepth(2).build().expand("mouse", WordNetPOS.NOUN); + + assertEquals( + lmf.stream().map(e -> e.term() + "|" + e.kind() + "|" + e.weight()).toList(), + wndb.stream().map(e -> e.term() + "|" + e.kind() + "|" + e.weight()).toList()); + assertNotNull(find(lmf, "rodent"), "got " + lmf); + } + + @Test + void testMorphyBridgesInflectedInput() { + final LexicalKnowledgeBase lexicon = WnLmfReaderTest.fixture(); + final LexicalExpander expander = LexicalExpander.builder(lexicon) + .lemmatizer(new MorphyLemmatizer(lexicon, MorphyExceptionsTest.fixture())) + .build(); + + // A regular inflection resolves by rule, an irregular one by the exception list. + final List dogs = expander.expand("dogs", WordNetPOS.NOUN); + assertEquals(Kind.SYNONYM, find(dogs, "dog").kind()); + assertNotNull(find(dogs, "canid"), "got " + dogs); + + final List mice = expander.expand("mice", WordNetPOS.NOUN); + assertEquals(Kind.SYNONYM, find(mice, "mouse").kind()); + assertNotNull(find(mice, "rodent"), "got " + mice); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java new file mode 100644 index 0000000000..d5fcad0360 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; +import opennlp.wordnet.LexicalExpander.Expansion; +import opennlp.wordnet.LexicalExpander.Kind; + +import static opennlp.wordnet.ExpansionAssertions.find; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Behavioral tests over a hand-built lexicon whose graph shape is fully controlled: sense + * ranking, hypernym depth and decay, hyponym opt-in, deduplication, exclusion of the input, + * cycle termination, and configuration validation. + */ +class LexicalExpanderTest { + + // dog: sense 1 = {dog, domestic dog} -> canid -> carnivore, with hyponym puppy; + // sense 2 = {dog, frank, hot dog} -> sausage. The verb sense = {dog, chase}. + // hot dog: the standalone multiword sense {hot dog, red hot}. + // alpha <-> beta form a malformed hypernym cycle. + // Lookups fold through LemmaFolding, exactly as the readers fold their keys at load time. + private static LexicalKnowledgeBase lexicon() { + final Map synsets = new HashMap<>(); + final Map> senses = new HashMap<>(); + + final Synset n1 = new Synset("n1", WordNetPOS.NOUN, List.of("dog", "domestic dog"), "canine", + Map.of(WordNetRelation.HYPERNYM, List.of("n2"), + WordNetRelation.HYPONYM, List.of("n4"))); + final Synset n2 = new Synset("n2", WordNetPOS.NOUN, List.of("canid"), "canid family", + Map.of(WordNetRelation.HYPERNYM, List.of("n3"))); + final Synset n3 = new Synset("n3", WordNetPOS.NOUN, List.of("carnivore"), "meat eater", + Map.of()); + final Synset n4 = new Synset("n4", WordNetPOS.NOUN, List.of("puppy"), "young dog", Map.of()); + final Synset n5 = new Synset("n5", WordNetPOS.NOUN, List.of("dog", "frank", "hot dog"), + "sausage in a bun", Map.of(WordNetRelation.HYPERNYM, List.of("n6"))); + final Synset n6 = new Synset("n6", WordNetPOS.NOUN, List.of("sausage"), "ground meat", + Map.of()); + final Synset v1 = new Synset("v1", WordNetPOS.VERB, List.of("dog", "chase"), "follow", + Map.of()); + final Synset m1 = new Synset("m1", WordNetPOS.NOUN, List.of("hot dog", "red hot"), + "grilled sausage", Map.of()); + final Synset c1 = new Synset("c1", WordNetPOS.NOUN, List.of("alpha"), "cycle start", + Map.of(WordNetRelation.HYPERNYM, List.of("c2"))); + final Synset c2 = new Synset("c2", WordNetPOS.NOUN, List.of("beta"), "cycle end", + Map.of(WordNetRelation.HYPERNYM, List.of("c1"))); + + for (final Synset synset : List.of(n1, n2, n3, n4, n5, n6, v1, m1, c1, c2)) { + synsets.put(synset.id(), synset); + } + senses.put("dog|NOUN", List.of(n1, n5)); + senses.put("dog|VERB", List.of(v1)); + senses.put("domestic dog|NOUN", List.of(n1)); + senses.put("hot dog|NOUN", List.of(m1)); + senses.put("alpha|NOUN", List.of(c1)); + + return new LexicalKnowledgeBase() { + @Override + public List lookup(String lemma, WordNetPOS pos) { + if (lemma == null || pos == null) { + throw new IllegalArgumentException("null"); + } + return senses.getOrDefault(LemmaFolding.fold(lemma) + "|" + pos, List.of()); + } + + @Override + public Optional synset(String synsetId) { + return Optional.ofNullable(synsets.get(synsetId)); + } + }; + } + + @Test + void testSynonymsAndHypernymsWithDefaultConfiguration() { + final List expansions = + LexicalExpander.builder(lexicon()).build().expand("dog", WordNetPOS.NOUN); + + final Expansion domesticDog = find(expansions, "domestic dog"); + assertEquals(Kind.SYNONYM, domesticDog.kind()); + assertEquals(1.0, domesticDog.weight()); + assertEquals(0, domesticDog.senseRank()); + + final Expansion canid = find(expansions, "canid"); + assertEquals(Kind.HYPERNYM, canid.kind()); + assertEquals(1, canid.depth()); + assertEquals(0.5, canid.weight()); + + final Expansion frank = find(expansions, "frank"); + assertEquals(Kind.SYNONYM, frank.kind()); + assertEquals(1, frank.senseRank()); + assertEquals(0.5, frank.weight()); + + // Depth 1 by default: the grandparent stays out, and so do hyponyms. + assertNull(find(expansions, "carnivore")); + assertNull(find(expansions, "puppy")); + } + + @Test + void testUnderscoreInputReachesTheSpaceFoldedLexiconEntry() { + // The lexicon keys "hot_dog" under its folded form "hot dog". The expander must fold the + // input the same way, so "hot dog" is excluded as the input itself and cannot displace the + // other member "red hot" from the capped result. + final List expansions = LexicalExpander.builder(lexicon()) + .maxExpansions(1).build().expand("hot_dog", WordNetPOS.NOUN); + + assertEquals(List.of(new Expansion("red hot", Kind.SYNONYM, 0, 0, 1.0)), expansions); + } + + @Test + void testTheInputTermIsNeverAnExpansion() { + for (final Expansion expansion : + LexicalExpander.builder(lexicon()).build().expand("dog", WordNetPOS.NOUN)) { + assertTrue(!expansion.term().equalsIgnoreCase("dog"), "got " + expansion); + } + } + + @Test + void testDeeperHypernymWalkDecaysPerStep() { + final List expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).build().expand("dog", WordNetPOS.NOUN); + + final Expansion carnivore = find(expansions, "carnivore"); + assertEquals(2, carnivore.depth()); + assertEquals(0.25, carnivore.weight()); + } + + @Test + void testHyponymsAreOptIn() { + final List expansions = LexicalExpander.builder(lexicon()) + .includeHyponyms(true).build().expand("dog", WordNetPOS.NOUN); + + final Expansion puppy = find(expansions, "puppy"); + assertEquals(Kind.HYPONYM, puppy.kind()); + assertEquals(0.5, puppy.weight()); + } + + @Test + void testMaxSensesLimitsToTheMostSalient() { + final List expansions = LexicalExpander.builder(lexicon()) + .maxSenses(1).build().expand("dog", WordNetPOS.NOUN); + + assertNull(find(expansions, "frank")); + assertNotNull(find(expansions, "domestic dog")); + } + + @Test + void testAllPosExpansionIncludesVerbSynonyms() { + final List expansions = + LexicalExpander.builder(lexicon()).build().expand("dog"); + + assertNotNull(find(expansions, "chase")); + assertNotNull(find(expansions, "domestic dog")); + } + + @Test + void testCyclicHypernymDataTerminates() { + final List expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(10).build().expand("alpha", WordNetPOS.NOUN); + + final Expansion beta = find(expansions, "beta"); + assertEquals(1, beta.depth()); + // The cycle leads back to alpha's own synset, which is visited and the input besides. + assertEquals(1, expansions.size()); + } + + @Test + void testDeduplicationKeepsTheHighestWeight() { + // "domestic dog" reaches n1 at rank 0; its synonym "dog" is the only other member and + // must appear once with the rank-0 weight even though deeper paths could yield it again. + final List expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).build().expand("domestic dog", WordNetPOS.NOUN); + + final Expansion dog = find(expansions, "dog"); + assertEquals(1.0, dog.weight()); + assertEquals(1, expansions.stream().filter(e -> e.term().equals("dog")).count()); + } + + @Test + void testOrderingIsWeightDescendingAndStable() { + final List expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).includeHyponyms(true).build().expand("dog", WordNetPOS.NOUN); + + for (int i = 1; i < expansions.size(); i++) { + assertTrue(expansions.get(i - 1).weight() >= expansions.get(i).weight(), + "weights must not increase: " + expansions); + } + assertEquals(expansions, + LexicalExpander.builder(lexicon()).hypernymDepth(2).includeHyponyms(true).build() + .expand("dog", WordNetPOS.NOUN)); + } + + @Test + void testMaxExpansionsCapsAfterRanking() { + final List expansions = LexicalExpander.builder(lexicon()) + .hypernymDepth(2).includeHyponyms(true).maxExpansions(2).build() + .expand("dog", WordNetPOS.NOUN); + + assertEquals(2, expansions.size()); + assertEquals(1.0, expansions.get(0).weight()); + } + + @Test + void testUnknownTermExpandsToNothing() { + assertEquals(List.of(), + LexicalExpander.builder(lexicon()).build().expand("xyzzy", WordNetPOS.NOUN)); + } + + @Test + void testLemmatizerFallbackExpandsInflectedInput() { + final LexicalExpander expander = LexicalExpander.builder(lexicon()) + .lemmatizer(new opennlp.tools.lemmatizer.Lemmatizer() { + @Override + public String[] lemmatize(String[] tokens, String[] tags) { + final String[] lemmas = new String[tokens.length]; + for (int i = 0; i < tokens.length; i++) { + lemmas[i] = "dogs".equals(tokens[i]) ? "dog" : "O"; + } + return lemmas; + } + + @Override + public List> lemmatize(List tokens, List tags) { + throw new UnsupportedOperationException(); + } + }) + .build(); + + final List expansions = expander.expand("dogs", WordNetPOS.NOUN); + // The lemma itself surfaces as a synonym, along with the rest of its synsets. + final Expansion dog = find(expansions, "dog"); + assertEquals(Kind.SYNONYM, dog.kind()); + assertEquals(1.0, dog.weight()); + assertNotNull(find(expansions, "domestic dog")); + + assertEquals(List.of(), expander.expand("cats", WordNetPOS.NOUN)); + } + + @Test + void testValidationFailsLoudly() { + assertThrows(IllegalArgumentException.class, () -> LexicalExpander.builder(null)); + final LexicalExpander.Builder builder = LexicalExpander.builder(lexicon()); + assertThrows(IllegalArgumentException.class, () -> builder.lemmatizer(null)); + assertThrows(IllegalArgumentException.class, () -> builder.maxSenses(0)); + assertThrows(IllegalArgumentException.class, () -> builder.hypernymDepth(-1)); + assertThrows(IllegalArgumentException.class, () -> builder.maxExpansions(0)); + assertThrows(IllegalArgumentException.class, () -> builder.senseDecay(0)); + assertThrows(IllegalArgumentException.class, () -> builder.senseDecay(1.5)); + assertThrows(IllegalArgumentException.class, () -> builder.depthDecay(0)); + assertThrows(IllegalArgumentException.class, () -> builder.depthDecay(1.5)); + + final LexicalExpander expander = builder.build(); + assertThrows(IllegalArgumentException.class, () -> expander.expand(null)); + assertThrows(IllegalArgumentException.class, () -> expander.expand(" ")); + assertThrows(IllegalArgumentException.class, () -> expander.expand("dog", null)); + } + + /** + * Verifies that decay products which underflow to zero are dropped instead of emitted: + * with the smallest positive depth decay, the first hypernym level keeps the smallest + * positive weight while the second level underflows to zero and never appears, and no + * reported expansion carries a weight outside the documented range. + */ + @Test + void testUnderflowedWeightsAreDropped() { + final List expansions = LexicalExpander.builder(lexicon()) + .depthDecay(Double.MIN_VALUE).hypernymDepth(2).maxSenses(1).build() + .expand("domestic dog", WordNetPOS.NOUN); + + final Expansion canid = find(expansions, "canid"); + assertNotNull(canid); + assertEquals(Double.MIN_VALUE, canid.weight(), 0.0); + assertNull(find(expansions, "carnivore")); + for (final Expansion expansion : expansions) { + assertTrue(expansion.weight() > 0.0 && expansion.weight() <= 1.0, + "weight out of (0, 1]: " + expansion); + } + } + + /** + * Verifies that the {@link Expansion} record rejects every component + * outside its documented range with a loud exception. + */ + @Test + void testExpansionValidatesItsComponents() { + assertThrows(IllegalArgumentException.class, () -> new Expansion( + null, Kind.SYNONYM, 0, 0, 1.0)); + assertThrows(IllegalArgumentException.class, () -> new Expansion( + " ", Kind.SYNONYM, 0, 0, 1.0)); + assertThrows(IllegalArgumentException.class, () -> new Expansion( + "dog", null, 0, 0, 1.0)); + assertThrows(IllegalArgumentException.class, () -> new Expansion( + "dog", Kind.SYNONYM, -1, 0, 1.0)); + assertThrows(IllegalArgumentException.class, () -> new Expansion( + "dog", Kind.SYNONYM, 0, -1, 1.0)); + assertThrows(IllegalArgumentException.class, () -> new Expansion( + "dog", Kind.SYNONYM, 0, 0, 0.0)); + assertThrows(IllegalArgumentException.class, () -> new Expansion( + "dog", Kind.SYNONYM, 0, 0, 1.5)); + assertThrows(IllegalArgumentException.class, () -> new Expansion( + "dog", Kind.SYNONYM, 0, 0, Double.NaN)); + } +} From 54697a9e1cccd9a4af3029a9c0d6a737e4d3058e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 09:12:14 -0400 Subject: [PATCH 08/15] OPENNLP-1887: Synset similarity measures and hypernym-anchored word typing SynsetSimilarity scores noun synset pairs with the path, Wu-Palmer, and Leacock-Chodorow measures over the knowledge base seam. HypernymTyper labels a word by walking its senses' hypernym and instance-hypernym chains to the nearest caller-registered anchor concept, so a knowledge base can type names as person, organization, or location without a model. Blank checks follow the toolkit whitespace definition. --- .gitignore | 1 + .../java/opennlp/wordnet/HypernymTyper.java | 163 ++++++++++++++ .../opennlp/wordnet/SynsetSimilarity.java | 202 ++++++++++++++++++ .../opennlp/wordnet/HypernymTyperTest.java | 132 ++++++++++++ .../opennlp/wordnet/SynsetSimilarityTest.java | 151 +++++++++++++ 5 files changed, 649 insertions(+) create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java diff --git a/.gitignore b/.gitignore index 965000f400..611df9b3c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.claude *.iml .idea target diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java new file mode 100644 index 0000000000..7a81e69ec7 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.wordnet; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import opennlp.tools.util.StringUtil; +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Types a noun by walking its hypernym chain to the nearest registered anchor: the + * caller names anchor concepts by lemma, {@code person}, {@code organization}, + * {@code location}, and any word whose senses lead up to an anchor's synsets receives + * that anchor's label. The nearest anchor wins, so a more specific registered concept + * beats a general one. + * + *

Anchors are resolved against the knowledge base at construction and follow its + * sense inventory; nothing beyond the caller's anchor choice is built in. Words with no + * sense reaching an anchor get no type.

+ * + *

The typer reads only immutable state and is safe to share between threads.

+ * + * @since 3.0.0 + */ +public class HypernymTyper { + + /** The relations that lead from a synset to its generalizations. */ + private static final List UPWARD_RELATIONS = + List.of(WordNetRelation.HYPERNYM, WordNetRelation.INSTANCE_HYPERNYM); + + private final LexicalKnowledgeBase knowledgeBase; + private final Map labelBySynsetId; + + /** + * Initializes the typer. + * + * @param knowledgeBase The knowledge base to walk. Must not be {@code null}. + * @param anchors The anchor lemmas mapped to the labels they confer, for example + * {@code person} to {@code person}. Every lemma is resolved as a noun; + * all its senses anchor. Must not be {@code null} or empty, and no + * lemma or label may be blank. + * @throws IllegalArgumentException Thrown if a parameter is {@code null}, + * {@code anchors} is empty or holds a blank entry, or an anchor lemma is + * unknown to the knowledge base. + */ + public HypernymTyper(LexicalKnowledgeBase knowledgeBase, Map anchors) { + if (knowledgeBase == null) { + throw new IllegalArgumentException("knowledgeBase must not be null"); + } + if (anchors == null || anchors.isEmpty()) { + throw new IllegalArgumentException("anchors must not be null or empty"); + } + this.knowledgeBase = knowledgeBase; + final Map labels = new LinkedHashMap<>(); + for (final Map.Entry anchor : anchors.entrySet()) { + if (anchor.getKey() == null || StringUtil.isBlank(anchor.getKey()) + || anchor.getValue() == null || StringUtil.isBlank(anchor.getValue())) { + throw new IllegalArgumentException("anchors must not contain blank entries"); + } + final List senses = knowledgeBase.lookup(anchor.getKey(), WordNetPOS.NOUN); + if (senses.isEmpty()) { + throw new IllegalArgumentException( + "anchor lemma is unknown to the knowledge base: " + anchor.getKey()); + } + for (final Synset sense : senses) { + labels.putIfAbsent(sense.id(), anchor.getValue()); + } + } + this.labelBySynsetId = Map.copyOf(labels); + } + + /** + * Types a noun by its nearest anchored hypernym. + * + * @param lemma The noun lemma to type. Must not be {@code null} or blank. + * @return The label of the nearest anchor over all senses, or empty when no sense + * reaches an anchor. + * @throws IllegalArgumentException Thrown if {@code lemma} is {@code null} or blank. + */ + public Optional type(String lemma) { + if (lemma == null || StringUtil.isBlank(lemma)) { + throw new IllegalArgumentException("lemma must not be null or blank"); + } + String bestLabel = null; + int bestDistance = Integer.MAX_VALUE; + for (final Synset sense : knowledgeBase.lookup(lemma, WordNetPOS.NOUN)) { + final int[] distance = new int[1]; + final String label = nearestAnchor(sense.id(), distance); + if (label != null && distance[0] < bestDistance) { + bestDistance = distance[0]; + bestLabel = label; + } + } + return Optional.ofNullable(bestLabel); + } + + /** + * Types a specific synset by its nearest anchored hypernym. + * + * @param synsetId The synset identifier. Must not be {@code null}. + * @return The nearest anchor's label, or empty when no ancestor is anchored. + * @throws IllegalArgumentException Thrown if {@code synsetId} is {@code null}. + */ + public Optional typeSynset(String synsetId) { + if (synsetId == null) { + throw new IllegalArgumentException("synsetId must not be null"); + } + return Optional.ofNullable(nearestAnchor(synsetId, new int[1])); + } + + /** Breadth-first walk up the hypernym graph to the closest anchored synset. */ + private String nearestAnchor(String synsetId, int[] distanceOut) { + final Set visited = new HashSet<>(); + final Deque queue = new ArrayDeque<>(); + final Map depths = new HashMap<>(); + queue.add(synsetId); + visited.add(synsetId); + depths.put(synsetId, 0); + while (!queue.isEmpty()) { + final String current = queue.remove(); + final String label = labelBySynsetId.get(current); + if (label != null) { + distanceOut[0] = depths.get(current); + return label; + } + for (final WordNetRelation relation : UPWARD_RELATIONS) { + for (final String parent : knowledgeBase.related(current, relation)) { + if (visited.add(parent)) { + depths.put(parent, depths.get(current) + 1); + queue.add(parent); + } + } + } + } + return null; + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java new file mode 100644 index 0000000000..72e682d236 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.wordnet; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Taxonomy-based similarity between synsets: measures over the hypernym graph of a + * {@link LexicalKnowledgeBase}, computed on demand with no precomputed tables. + * + *

Path similarity is {@code 1 / (1 + d)} for the shortest hypernym-graph distance + * {@code d} through a common ancestor. Wu-Palmer similarity relates the depth of the + * deepest common ancestor to the depths of both synsets. Leacock-Chodorow scales the + * shortest path against a caller-supplied taxonomy depth, since the knowledge base + * interface does not enumerate the taxonomy. Unrelated synsets, those sharing no + * ancestor, score zero everywhere. Information-content measures need corpus counts and + * are not provided here.

+ * + *

Both plain and instance hypernyms count as taxonomy edges. The measures read only + * the knowledge base and hold no mutable state, so instances are as thread-safe as + * their knowledge base.

+ * + * @since 3.0.0 + */ +public class SynsetSimilarity { + + private final LexicalKnowledgeBase knowledgeBase; + + /** + * Initializes the measures. + * + * @param knowledgeBase The knowledge base to walk. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code knowledgeBase} is {@code null}. + */ + public SynsetSimilarity(LexicalKnowledgeBase knowledgeBase) { + if (knowledgeBase == null) { + throw new IllegalArgumentException("knowledgeBase must not be null"); + } + this.knowledgeBase = knowledgeBase; + } + + /** + * Computes path similarity: {@code 1 / (1 + d)} over the shortest hypernym-graph + * distance. + * + * @param synsetId The first synset identifier. Must not be {@code null}. + * @param otherSynsetId The second synset identifier. Must not be {@code null}. + * @return The similarity in {@code (0, 1]}, or {@code 0} when the synsets share no + * ancestor. + * @throws IllegalArgumentException Thrown if an identifier is {@code null}. + */ + public double path(String synsetId, String otherSynsetId) { + final int distance = shortestDistance(synsetId, otherSynsetId); + return distance < 0 ? 0.0 : 1.0 / (1.0 + distance); + } + + /** + * Computes Wu-Palmer similarity: {@code 2 * depth(lcs) / (depth(a) + depth(b))}, + * with depths counted from the taxonomy root and the deepest common ancestor as the + * lcs. + * + * @param synsetId The first synset identifier. Must not be {@code null}. + * @param otherSynsetId The second synset identifier. Must not be {@code null}. + * @return The similarity in {@code (0, 1]}, or {@code 0} when the synsets share no + * ancestor. + * @throws IllegalArgumentException Thrown if an identifier is {@code null}. + */ + public double wuPalmer(String synsetId, String otherSynsetId) { + final Map up = depthsAbove(synsetId); + final Map otherUp = depthsAbove(otherSynsetId); + double best = 0.0; + for (final Map.Entry common : up.entrySet()) { + final Integer otherDistance = otherUp.get(common.getKey()); + if (otherDistance == null) { + continue; + } + final int rootDepth = depthFromRoot(common.getKey()); + final int depthA = rootDepth + common.getValue(); + final int depthB = rootDepth + otherDistance; + if (depthA + depthB == 0) { + continue; + } + final double score = 2.0 * rootDepth / (depthA + depthB); + best = Math.max(best, score); + } + return best; + } + + /** + * Computes Leacock-Chodorow similarity: + * {@code -log((d + 1) / (2 * taxonomyDepth))} over the shortest hypernym-graph + * distance {@code d}. + * + * @param synsetId The first synset identifier. Must not be {@code null}. + * @param otherSynsetId The second synset identifier. Must not be {@code null}. + * @param taxonomyDepth The maximum depth of the taxonomy the synsets live in. Must + * be positive. + * @return The similarity, higher for closer synsets, or {@code 0} when the synsets + * share no ancestor. + * @throws IllegalArgumentException Thrown if an identifier is {@code null} or + * {@code taxonomyDepth} is not positive. + */ + public double leacockChodorow(String synsetId, String otherSynsetId, + int taxonomyDepth) { + if (taxonomyDepth <= 0) { + throw new IllegalArgumentException( + "taxonomyDepth must be positive: " + taxonomyDepth); + } + final int distance = shortestDistance(synsetId, otherSynsetId); + if (distance < 0) { + return 0.0; + } + return -Math.log((distance + 1.0) / (2.0 * taxonomyDepth)); + } + + /** + * Finds the shortest distance between two synsets through a common ancestor. + * + * @param synsetId The first synset identifier. Must not be {@code null}. + * @param otherSynsetId The second synset identifier. Must not be {@code null}. + * @return The edge count of the shortest connecting path, or {@code -1} when no + * common ancestor exists. + * @throws IllegalArgumentException Thrown if an identifier is {@code null}. + */ + public int shortestDistance(String synsetId, String otherSynsetId) { + final Map up = depthsAbove(synsetId); + final Map otherUp = depthsAbove(otherSynsetId); + int best = -1; + for (final Map.Entry common : up.entrySet()) { + final Integer otherDistance = otherUp.get(common.getKey()); + if (otherDistance != null) { + final int total = common.getValue() + otherDistance; + if (best < 0 || total < best) { + best = total; + } + } + } + return best; + } + + /** Collects every ancestor with its minimal upward distance, the synset included. */ + private Map depthsAbove(String synsetId) { + if (synsetId == null) { + throw new IllegalArgumentException("synset identifiers must not be null"); + } + final Map depths = new HashMap<>(); + final Deque queue = new ArrayDeque<>(); + depths.put(synsetId, 0); + queue.add(synsetId); + while (!queue.isEmpty()) { + final String current = queue.remove(); + final int depth = depths.get(current); + for (final String parent : hypernyms(current)) { + if (!depths.containsKey(parent) || depths.get(parent) > depth + 1) { + depths.put(parent, depth + 1); + queue.add(parent); + } + } + } + return depths; + } + + /** Measures a synset's depth from its taxonomy root, the shortest way up. */ + private int depthFromRoot(String synsetId) { + final Map above = depthsAbove(synsetId); + int deepest = 0; + for (final int distance : above.values()) { + deepest = Math.max(deepest, distance); + } + return deepest; + } + + private Iterable hypernyms(String synsetId) { + final List parents = new ArrayList<>( + knowledgeBase.related(synsetId, WordNetRelation.HYPERNYM)); + parents.addAll(knowledgeBase.related(synsetId, WordNetRelation.INSTANCE_HYPERNYM)); + return parents; + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java new file mode 100644 index 0000000000..618c4d885d --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.wordnet; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Tests that {@link HypernymTyper} labels a word by its nearest anchored hypernym over + * the fixture taxonomy of {@link SynsetSimilarityTest}, follows instance hypernymy, + * prefers the closer of two anchors, and validates its arguments. + */ +public class HypernymTyperTest { + + /** + * @return A typer over the shared fixture taxonomy with person and location anchors. + * Never {@code null}. + */ + private static HypernymTyper typer() { + return new HypernymTyper(taxonomy(), + Map.of("person", "person", "location", "location")); + } + + /** + * @return The shared fixture taxonomy. Never {@code null}. + */ + private static SynsetSimilarityTest.FixtureKnowledgeBase taxonomy() { + final SynsetSimilarityTest.FixtureKnowledgeBase kb = + new SynsetSimilarityTest.FixtureKnowledgeBase(); + kb.add("n1", "entity", WordNetRelation.HYPERNYM); + kb.add("n2", "physical", WordNetRelation.HYPERNYM, "n1"); + kb.add("n3", "organism", WordNetRelation.HYPERNYM, "n2"); + kb.add("n4", "person", WordNetRelation.HYPERNYM, "n3"); + kb.add("n5", "scientist", WordNetRelation.HYPERNYM, "n4"); + kb.add("n6", "chemist", WordNetRelation.HYPERNYM, "n5"); + kb.add("n7", "location", WordNetRelation.HYPERNYM, "n2"); + kb.add("n8", "city", WordNetRelation.HYPERNYM, "n7"); + kb.add("n11", "paris", WordNetRelation.INSTANCE_HYPERNYM, "n8"); + kb.add("n12", "abstract", WordNetRelation.HYPERNYM); + return kb; + } + + /** + * Verifies that a noun whose hypernym chain reaches an anchor receives that anchor's + * label, both through plain and instance hypernymy, and that the anchor lemma itself + * is typed with its own label at distance zero. + */ + @Test + void testTypesThroughHypernymAndInstanceChains() { + final HypernymTyper typer = typer(); + Assertions.assertEquals(Optional.of("person"), typer.type("chemist")); + Assertions.assertEquals(Optional.of("location"), typer.type("paris")); + Assertions.assertEquals(Optional.of("person"), typer.type("person")); + Assertions.assertEquals(Optional.of("location"), typer.typeSynset("n8")); + } + + /** + * Verifies that no label is produced when no sense of the word, or no ancestor of + * the synset, reaches an anchor. + */ + @Test + void testUnreachableAnchorsYieldEmpty() { + final HypernymTyper typer = typer(); + Assertions.assertEquals(Optional.empty(), typer.type("abstract")); + Assertions.assertEquals(Optional.empty(), typer.type("unknownword")); + Assertions.assertEquals(Optional.empty(), typer.typeSynset("n12")); + Assertions.assertEquals(Optional.empty(), typer.typeSynset("missing")); + } + + /** + * Verifies that the nearest anchor wins: with scientist registered as its own + * anchor, a chemist is a scientist rather than the more distant person. + */ + @Test + void testNearestAnchorWins() { + final Map anchors = new LinkedHashMap<>(); + anchors.put("person", "person"); + anchors.put("scientist", "scientist"); + final HypernymTyper typer = new HypernymTyper(taxonomy(), anchors); + Assertions.assertEquals(Optional.of("scientist"), typer.type("chemist")); + Assertions.assertEquals(Optional.of("scientist"), typer.type("scientist")); + // the walk is upward only, so an ancestor of an anchor is never typed by it + Assertions.assertEquals(Optional.empty(), typer.type("organism")); + } + + /** + * Verifies that invalid construction and query arguments are rejected: null or + * empty inputs, blank anchor entries, and an anchor lemma the knowledge base does + * not know. + */ + @Test + void testInvalidArguments() { + final Map anchors = Map.of("person", "person"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(null, anchors)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(taxonomy(), null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(taxonomy(), Map.of())); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(taxonomy(), Map.of(" ", "person"))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(taxonomy(), Map.of("person", " "))); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(taxonomy(), Map.of("notaword", "label"))); + final HypernymTyper typer = typer(); + Assertions.assertThrows(IllegalArgumentException.class, () -> typer.type(null)); + Assertions.assertThrows(IllegalArgumentException.class, () -> typer.type(" ")); + Assertions.assertThrows(IllegalArgumentException.class, () -> typer.typeSynset(null)); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java new file mode 100644 index 0000000000..3264970585 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.wordnet; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Tests the taxonomy measures and the hypernym typer against a project-authored + * miniature taxonomy; no external lexicon data is involved. + */ +public class SynsetSimilarityTest { + + /** A tiny in-memory knowledge base over a hand-built noun taxonomy. */ + static final class FixtureKnowledgeBase implements LexicalKnowledgeBase { + private final Map byId = new HashMap<>(); + private final Map> byLemma = new HashMap<>(); + + void add(String id, String lemma, WordNetRelation relation, String... parents) { + final Map> relations = parents.length == 0 + ? Map.of() : Map.of(relation, List.of(parents)); + final Synset synset = + new Synset(id, WordNetPOS.NOUN, List.of(lemma), "fixture", relations); + byId.put(id, synset); + byLemma.computeIfAbsent(lemma, key -> new java.util.ArrayList<>()).add(synset); + } + + @Override + public List lookup(String lemma, WordNetPOS pos) { + return byLemma.getOrDefault(lemma, List.of()); + } + + @Override + public Optional synset(String synsetId) { + return Optional.ofNullable(byId.get(synsetId)); + } + } + + private static FixtureKnowledgeBase taxonomy() { + final FixtureKnowledgeBase kb = new FixtureKnowledgeBase(); + kb.add("n1", "entity", WordNetRelation.HYPERNYM); + kb.add("n2", "physical", WordNetRelation.HYPERNYM, "n1"); + kb.add("n3", "organism", WordNetRelation.HYPERNYM, "n2"); + kb.add("n4", "person", WordNetRelation.HYPERNYM, "n3"); + kb.add("n5", "scientist", WordNetRelation.HYPERNYM, "n4"); + kb.add("n6", "chemist", WordNetRelation.HYPERNYM, "n5"); + kb.add("n7", "location", WordNetRelation.HYPERNYM, "n2"); + kb.add("n8", "city", WordNetRelation.HYPERNYM, "n7"); + kb.add("n9", "organization", WordNetRelation.HYPERNYM, "n1"); + kb.add("n10", "company", WordNetRelation.HYPERNYM, "n9"); + kb.add("n11", "paris", WordNetRelation.INSTANCE_HYPERNYM, "n8"); + kb.add("n12", "abstract", WordNetRelation.HYPERNYM); + return kb; + } + + @Test + void testPathSimilarity() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + Assertions.assertEquals(1.0, similarity.path("n5", "n5"), 1e-9); + Assertions.assertEquals(0.5, similarity.path("n6", "n5"), 1e-9); + // chemist up four to physical, city up two: six edges apart + Assertions.assertEquals(1.0 / 7.0, similarity.path("n6", "n8"), 1e-9); + Assertions.assertEquals(0.0, similarity.path("n6", "n12"), 1e-9); + } + + @Test + void testWuPalmerRewardsDeepSharedAncestry() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + // scientist and chemist share scientist itself at depth four + Assertions.assertEquals(8.0 / 9.0, similarity.wuPalmer("n5", "n6"), 1e-9); + final double siblingBranches = similarity.wuPalmer("n6", "n8"); + Assertions.assertTrue(siblingBranches < similarity.wuPalmer("n5", "n6")); + Assertions.assertTrue(siblingBranches > 0.0); + Assertions.assertEquals(0.0, similarity.wuPalmer("n6", "n12"), 1e-9); + } + + @Test + void testLeacockChodorow() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + Assertions.assertEquals(Math.log(10.0), + similarity.leacockChodorow("n5", "n6", 10), 1e-9); + Assertions.assertEquals(0.0, similarity.leacockChodorow("n6", "n12", 10), 1e-9); + Assertions.assertThrows(IllegalArgumentException.class, + () -> similarity.leacockChodorow("n5", "n6", 0)); + } + + @Test + void testInstanceHypernymsCountAsEdges() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + Assertions.assertEquals(0.5, similarity.path("n11", "n8"), 1e-9); + } + + @Test + void testTyperFindsTheNearestAnchor() { + final HypernymTyper typer = new HypernymTyper(taxonomy(), Map.of( + "person", "person", "location", "location", "organization", "organization")); + Assertions.assertEquals("person", typer.type("chemist").orElseThrow()); + Assertions.assertEquals("location", typer.type("city").orElseThrow()); + Assertions.assertEquals("organization", typer.type("company").orElseThrow()); + Assertions.assertEquals("location", typer.typeSynset("n11").orElseThrow()); + Assertions.assertTrue(typer.type("entity").isEmpty()); + Assertions.assertTrue(typer.type("blorp").isEmpty()); + } + + @Test + void testMoreSpecificAnchorsWin() { + final HypernymTyper typer = new HypernymTyper(taxonomy(), Map.of( + "person", "person", "scientist", "researcher")); + Assertions.assertEquals("researcher", typer.type("chemist").orElseThrow()); + Assertions.assertEquals("person", typer.type("person").orElseThrow()); + } + + @Test + void testInvalidArguments() { + Assertions.assertThrows(IllegalArgumentException.class, + () -> new SynsetSimilarity(null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new SynsetSimilarity(taxonomy()).path(null, "n1")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(taxonomy(), Map.of())); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new HypernymTyper(taxonomy(), Map.of("blorp", "thing"))); + final HypernymTyper typer = new HypernymTyper(taxonomy(), Map.of("person", "person")); + Assertions.assertThrows(IllegalArgumentException.class, () -> typer.type(" ")); + } +} From 7eeadf52d7b1f9a9f215f104bdafe55587067fdc Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:40:23 -0400 Subject: [PATCH 09/15] OPENNLP-1887: Extend the WordNet manual with mirror-tested expansion examples Add the lexical expansion section to docbkx/wordnet.xml and LexicalExpansionUsageExampleTest asserting the expansion values the chapter prints; carry the WordNet usage example alongside it. --- opennlp-docs/src/docbkx/wordnet.xml | 49 +++++- .../LexicalExpansionUsageExampleTest.java | 158 ++++++++++++++++++ 2 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java diff --git a/opennlp-docs/src/docbkx/wordnet.xml b/opennlp-docs/src/docbkx/wordnet.xml index e3db734d15..82d5e910ca 100644 --- a/opennlp-docs/src/docbkx/wordnet.xml +++ b/opennlp-docs/src/docbkx/wordnet.xml @@ -25,7 +25,9 @@ of speech. Two readers are provided: WnLmfReader for the Global WordNet Association WN-LMF XML interchange format, and WndbReader for the classic Princeton WordNet database file - layout. Both return an immutable, thread-safe knowledge base. + layout. Both return an immutable, thread-safe knowledge base. On top of + lookup, the module can Morphy-lemmatize, expand a term through synonym and + hypernym links, and score synset similarity on the hypernym graph.
@@ -103,4 +105,49 @@ lemmatizer.lemmatize(new String[] {"dogs"}, new String[] {"NNS"})[0]; // "dog"] + +
+ Lexical expansion + + LexicalExpander turns a term into related terms from the + knowledge base: synonyms sharing its synsets, hypernym ancestors up to a + configured depth, and optionally direct hyponyms. Each + Expansion carries a heuristic weight in + (0, 1]: the first sense starts at 1.0, later + senses multiply by the sense decay, and each hypernym or hyponym step + multiplies by the depth decay. The input term itself is never returned. + Defaults use depth 1, sense decay 0.5, and + depth decay 0.5. + LexicalExpansionUsageExampleTest asserts the behavior shown + here. + expansions = LexicalExpander.builder(lexicon) + .build() + .expand("dog", WordNetPOS.NOUN); + +// "domestic dog": SYNONYM, senseRank 0, weight 1.0 +// "canid": HYPERNYM, depth 1, weight 0.5 +// "frank": SYNONYM, senseRank 1, weight 0.5]]> + + +
+ +
+ Synset similarity + + SynsetSimilarity scores two synset identifiers on the + hypernym graph. Path similarity is 1 / (1 + d) for the + shortest distance d through a common ancestor. Wu-Palmer + similarity relates the depth of the deepest common ancestor to the depths + of both synsets. Unrelated synsets score 0. + LexicalExpansionUsageExampleTest asserts the behavior shown + here. + + + +
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java new file mode 100644 index 0000000000..ace8265cd6 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.wordnet; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.LexicalKnowledgeBase; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; +import opennlp.wordnet.LexicalExpander.Expansion; +import opennlp.wordnet.LexicalExpander.Kind; + +/** + * Runs the manual's lexical expansion and synset similarity examples (docbkx + * {@code wordnet.xml}) verbatim: every value the chapter states is asserted here, so a + * change breaking this test breaks the manual. The taxonomy is a hand-built miniature + * matching the shapes used elsewhere in this module's tests. + */ +public class LexicalExpansionUsageExampleTest { + + /** + * dog sense 1 = {dog, domestic dog} -> canid; sense 2 = {dog, frank, hot dog} -> sausage. + */ + private static LexicalKnowledgeBase dogTaxonomy() { + final Map synsets = new HashMap<>(); + final Map> senses = new HashMap<>(); + + final Synset n1 = new Synset("n1", WordNetPOS.NOUN, List.of("dog", "domestic dog"), "canine", + Map.of(WordNetRelation.HYPERNYM, List.of("n2"))); + final Synset n2 = new Synset("n2", WordNetPOS.NOUN, List.of("canid"), "canid family", Map.of()); + final Synset n5 = new Synset("n5", WordNetPOS.NOUN, List.of("dog", "frank", "hot dog"), + "sausage in a bun", Map.of(WordNetRelation.HYPERNYM, List.of("n6"))); + final Synset n6 = new Synset("n6", WordNetPOS.NOUN, List.of("sausage"), "ground meat", + Map.of()); + + for (final Synset synset : List.of(n1, n2, n5, n6)) { + synsets.put(synset.id(), synset); + } + senses.put("dog|NOUN", List.of(n1, n5)); + senses.put("domestic dog|NOUN", List.of(n1)); + + return new LexicalKnowledgeBase() { + @Override + public List lookup(String lemma, WordNetPOS pos) { + if (lemma == null || pos == null) { + throw new IllegalArgumentException("lemma and pos must not be null"); + } + return senses.getOrDefault(LemmaFolding.fold(lemma) + "|" + pos, List.of()); + } + + @Override + public Optional synset(String synsetId) { + return Optional.ofNullable(synsets.get(synsetId)); + } + }; + } + + /** + * chemist -> scientist -> person -> organism -> physical -> entity; city -> location -> + * physical. + */ + private static LexicalKnowledgeBase similarityTaxonomy() { + final Map byId = new HashMap<>(); + add(byId, "n1", "entity"); + add(byId, "n2", "physical", "n1"); + add(byId, "n3", "organism", "n2"); + add(byId, "n4", "person", "n3"); + add(byId, "n5", "scientist", "n4"); + add(byId, "n6", "chemist", "n5"); + add(byId, "n7", "location", "n2"); + add(byId, "n8", "city", "n7"); + return new LexicalKnowledgeBase() { + @Override + public List lookup(String lemma, WordNetPOS pos) { + return List.of(); + } + + @Override + public Optional synset(String synsetId) { + return Optional.ofNullable(byId.get(synsetId)); + } + }; + } + + private static void add(Map byId, String id, String lemma, String... parents) { + final Map> relations = parents.length == 0 + ? Map.of() : Map.of(WordNetRelation.HYPERNYM, List.of(parents)); + byId.put(id, new Synset(id, WordNetPOS.NOUN, List.of(lemma), "fixture", relations)); + } + + private static Expansion find(List expansions, String term) { + for (final Expansion expansion : expansions) { + if (term.equals(expansion.term())) { + return expansion; + } + } + return null; + } + + /** + * Default expansion of noun {@code dog}: synonym and depth-1 hypernym weights. + */ + @Test + void testExpandDogNoun() { + final List expansions = + LexicalExpander.builder(dogTaxonomy()).build().expand("dog", WordNetPOS.NOUN); + + final Expansion domesticDog = find(expansions, "domestic dog"); + Assertions.assertNotNull(domesticDog); + Assertions.assertEquals(Kind.SYNONYM, domesticDog.kind()); + Assertions.assertEquals(1.0, domesticDog.weight()); + Assertions.assertEquals(0, domesticDog.senseRank()); + + final Expansion canid = find(expansions, "canid"); + Assertions.assertNotNull(canid); + Assertions.assertEquals(Kind.HYPERNYM, canid.kind()); + Assertions.assertEquals(1, canid.depth()); + Assertions.assertEquals(0.5, canid.weight()); + + final Expansion frank = find(expansions, "frank"); + Assertions.assertNotNull(frank); + Assertions.assertEquals(Kind.SYNONYM, frank.kind()); + Assertions.assertEquals(1, frank.senseRank()); + Assertions.assertEquals(0.5, frank.weight()); + } + + /** + * Path and Wu-Palmer scores on the miniature scientist/city taxonomy. + */ + @Test + void testSynsetSimilarityScores() { + final SynsetSimilarity similarity = new SynsetSimilarity(similarityTaxonomy()); + Assertions.assertEquals(0.5, similarity.path("n6", "n5"), 1e-9); + Assertions.assertEquals(8.0 / 9.0, similarity.wuPalmer("n5", "n6"), 1e-9); + } +} From 8af138f3920c42b7319b786a83a58595341bff85 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 06:50:35 -0400 Subject: [PATCH 10/15] OPENNLP-1887: Align null contracts, annotations, and dev helper placement with the review conventions --- .../java/opennlp/wordnet/HypernymTyper.java | 4 +- .../java/opennlp/wordnet/LexicalExpander.java | 5 ++- .../opennlp/wordnet/SynsetSimilarity.java | 40 ++++++++++++++----- .../opennlp/wordnet/HypernymTyperTest.java | 2 +- .../opennlp/wordnet/LexicalExpanderTest.java | 3 +- .../LexicalExpansionUsageExampleTest.java | 7 +++- 6 files changed, 43 insertions(+), 18 deletions(-) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java index 7a81e69ec7..194a4e09ee 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java @@ -27,6 +27,7 @@ import java.util.Optional; import java.util.Set; +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.util.StringUtil; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; @@ -45,9 +46,8 @@ * sense reaching an anchor get no type.

* *

The typer reads only immutable state and is safe to share between threads.

- * - * @since 3.0.0 */ +@ThreadSafe public class HypernymTyper { /** The relations that lead from a synset to its generalizations. */ diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java index b21f45126f..cc604dce39 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java @@ -27,6 +27,7 @@ import opennlp.tools.commons.ThreadSafe; import opennlp.tools.lemmatizer.Lemmatizer; +import opennlp.tools.util.StringUtil; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; import opennlp.tools.wordnet.WordNetPOS; @@ -91,7 +92,7 @@ public record Expansion(String term, Kind kind, int depth, int senseRank, double * {@code weight} is not in {@code (0, 1]}. */ public Expansion { - if (term == null || term.isBlank()) { + if (term == null || StringUtil.isBlank(term)) { throw new IllegalArgumentException("term must not be null or blank"); } if (kind == null) { @@ -184,7 +185,7 @@ public List expand(String term) { * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or blank. */ private List collect(String term, List poses) { - if (term == null || term.isBlank()) { + if (term == null || StringUtil.isBlank(term)) { throw new IllegalArgumentException("The term must not be null or blank."); } final Map best = new HashMap<>(); diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java index 72e682d236..d0a28796fb 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.WordNetRelation; @@ -42,9 +43,8 @@ *

Both plain and instance hypernyms count as taxonomy edges. The measures read only * the knowledge base and hold no mutable state, so instances are as thread-safe as * their knowledge base.

- * - * @since 3.0.0 */ +@ThreadSafe public class SynsetSimilarity { private final LexicalKnowledgeBase knowledgeBase; @@ -70,7 +70,8 @@ public SynsetSimilarity(LexicalKnowledgeBase knowledgeBase) { * @param otherSynsetId The second synset identifier. Must not be {@code null}. * @return The similarity in {@code (0, 1]}, or {@code 0} when the synsets share no * ancestor. - * @throws IllegalArgumentException Thrown if an identifier is {@code null}. + * @throws IllegalArgumentException Thrown if {@code synsetId} or + * {@code otherSynsetId} is {@code null}. */ public double path(String synsetId, String otherSynsetId) { final int distance = shortestDistance(synsetId, otherSynsetId); @@ -86,9 +87,11 @@ public double path(String synsetId, String otherSynsetId) { * @param otherSynsetId The second synset identifier. Must not be {@code null}. * @return The similarity in {@code (0, 1]}, or {@code 0} when the synsets share no * ancestor. - * @throws IllegalArgumentException Thrown if an identifier is {@code null}. + * @throws IllegalArgumentException Thrown if {@code synsetId} or + * {@code otherSynsetId} is {@code null}. */ public double wuPalmer(String synsetId, String otherSynsetId) { + validateIds(synsetId, otherSynsetId); final Map up = depthsAbove(synsetId); final Map otherUp = depthsAbove(otherSynsetId); double best = 0.0; @@ -120,8 +123,9 @@ public double wuPalmer(String synsetId, String otherSynsetId) { * be positive. * @return The similarity, higher for closer synsets, or {@code 0} when the synsets * share no ancestor. - * @throws IllegalArgumentException Thrown if an identifier is {@code null} or - * {@code taxonomyDepth} is not positive. + * @throws IllegalArgumentException Thrown if {@code synsetId} or + * {@code otherSynsetId} is {@code null}, or {@code taxonomyDepth} is not + * positive. */ public double leacockChodorow(String synsetId, String otherSynsetId, int taxonomyDepth) { @@ -143,9 +147,11 @@ public double leacockChodorow(String synsetId, String otherSynsetId, * @param otherSynsetId The second synset identifier. Must not be {@code null}. * @return The edge count of the shortest connecting path, or {@code -1} when no * common ancestor exists. - * @throws IllegalArgumentException Thrown if an identifier is {@code null}. + * @throws IllegalArgumentException Thrown if {@code synsetId} or + * {@code otherSynsetId} is {@code null}. */ public int shortestDistance(String synsetId, String otherSynsetId) { + validateIds(synsetId, otherSynsetId); final Map up = depthsAbove(synsetId); final Map otherUp = depthsAbove(otherSynsetId); int best = -1; @@ -161,11 +167,25 @@ public int shortestDistance(String synsetId, String otherSynsetId) { return best; } - /** Collects every ancestor with its minimal upward distance, the synset included. */ - private Map depthsAbove(String synsetId) { + /** + * Validates the identifier arguments of the public measures. + * + * @param synsetId The first synset identifier. + * @param otherSynsetId The second synset identifier. + * @throws IllegalArgumentException Thrown if {@code synsetId} or + * {@code otherSynsetId} is {@code null}. + */ + private void validateIds(String synsetId, String otherSynsetId) { if (synsetId == null) { - throw new IllegalArgumentException("synset identifiers must not be null"); + throw new IllegalArgumentException("synsetId must not be null"); } + if (otherSynsetId == null) { + throw new IllegalArgumentException("otherSynsetId must not be null"); + } + } + + /** Collects every ancestor with its minimal upward distance, the synset included. */ + private Map depthsAbove(String synsetId) { final Map depths = new HashMap<>(); final Deque queue = new ArrayDeque<>(); depths.put(synsetId, 0); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java index 618c4d885d..5c14a1c554 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java @@ -121,7 +121,7 @@ void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, () -> new HypernymTyper(taxonomy(), Map.of(" ", "person"))); Assertions.assertThrows(IllegalArgumentException.class, - () -> new HypernymTyper(taxonomy(), Map.of("person", " "))); + () -> new HypernymTyper(taxonomy(), Map.of("person", "\u00A0"))); Assertions.assertThrows(IllegalArgumentException.class, () -> new HypernymTyper(taxonomy(), Map.of("notaword", "label"))); final HypernymTyper typer = typer(); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java index d5fcad0360..91da149935 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java @@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test; +import opennlp.tools.lemmatizer.Lemmatizer; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; import opennlp.tools.wordnet.WordNetPOS; @@ -237,7 +238,7 @@ void testUnknownTermExpandsToNothing() { @Test void testLemmatizerFallbackExpandsInflectedInput() { final LexicalExpander expander = LexicalExpander.builder(lexicon()) - .lemmatizer(new opennlp.tools.lemmatizer.Lemmatizer() { + .lemmatizer(new Lemmatizer() { @Override public String[] lemmatize(String[] tokens, String[] tags) { final String[] lemmas = new String[tokens.length]; diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java index ace8265cd6..a379b7a0c3 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java @@ -64,8 +64,11 @@ private static LexicalKnowledgeBase dogTaxonomy() { return new LexicalKnowledgeBase() { @Override public List lookup(String lemma, WordNetPOS pos) { - if (lemma == null || pos == null) { - throw new IllegalArgumentException("lemma and pos must not be null"); + if (lemma == null) { + throw new IllegalArgumentException("lemma must not be null"); + } + if (pos == null) { + throw new IllegalArgumentException("pos must not be null"); } return senses.getOrDefault(LemmaFolding.fold(lemma) + "|" + pos, List.of()); } From b1a9a2a557d76d34d83a79536e0f4fdfae03f449 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:01:36 -0400 Subject: [PATCH 11/15] OPENNLP-1887: Address review: expander and similarity javadoc, pinning tests --- .../java/opennlp/wordnet/HypernymTyper.java | 18 +++--- .../java/opennlp/wordnet/LexicalExpander.java | 50 ++++++++--------- .../opennlp/wordnet/SynsetSimilarity.java | 23 +++++++- .../opennlp/wordnet/HypernymTyperTest.java | 18 +----- .../wordnet/LexicalExpanderLexiconTest.java | 2 +- .../opennlp/wordnet/LexicalExpanderTest.java | 55 ++++++++++++------- .../LexicalExpansionUsageExampleTest.java | 11 +--- .../opennlp/wordnet/SynsetSimilarityTest.java | 52 ++++++++---------- 8 files changed, 120 insertions(+), 109 deletions(-) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java index 194a4e09ee..02756d0e1e 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/HypernymTyper.java @@ -20,12 +20,10 @@ import java.util.ArrayDeque; import java.util.Deque; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import opennlp.tools.commons.ThreadSafe; import opennlp.tools.util.StringUtil; @@ -134,13 +132,19 @@ public Optional typeSynset(String synsetId) { return Optional.ofNullable(nearestAnchor(synsetId, new int[1])); } - /** Breadth-first walk up the hypernym graph to the closest anchored synset. */ + /** + * Walks up the hypernym graph breadth first to the closest anchored synset. Visiting each + * synset once bounds the walk even on cyclic data. + * + * @param synsetId The synset to start from. Must not be {@code null}. + * @param distanceOut A single-element array that receives the edge count to the anchor found; + * left untouched when no ancestor is anchored. + * @return The label of the nearest anchored synset, or {@code null} when none is reachable. + */ private String nearestAnchor(String synsetId, int[] distanceOut) { - final Set visited = new HashSet<>(); final Deque queue = new ArrayDeque<>(); final Map depths = new HashMap<>(); queue.add(synsetId); - visited.add(synsetId); depths.put(synsetId, 0); while (!queue.isEmpty()) { final String current = queue.remove(); @@ -149,10 +153,10 @@ private String nearestAnchor(String synsetId, int[] distanceOut) { distanceOut[0] = depths.get(current); return label; } + final int parentDepth = depths.get(current) + 1; for (final WordNetRelation relation : UPWARD_RELATIONS) { for (final String parent : knowledgeBase.related(current, relation)) { - if (visited.add(parent)) { - depths.put(parent, depths.get(current) + 1); + if (depths.putIfAbsent(parent, parentDepth) == null) { queue.add(parent); } } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java index cc604dce39..5a110cac84 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LexicalExpander.java @@ -138,9 +138,9 @@ private LexicalExpander(Builder builder) { /** * Starts a builder. * - * @param lexicon The knowledge base to expand against; must not be null. + * @param lexicon The knowledge base to expand against. Must not be {@code null}. * @return A builder with the default configuration. - * @throws IllegalArgumentException Thrown if {@code lexicon} is null. + * @throws IllegalArgumentException Thrown if {@code lexicon} is {@code null}. */ public static Builder builder(LexicalKnowledgeBase lexicon) { return new Builder(lexicon); @@ -149,16 +149,16 @@ public static Builder builder(LexicalKnowledgeBase lexicon) { /** * Expands a term for one part of speech. * - * @param term The term to expand; must not be null or blank. - * @param pos The part of speech to expand as; must not be null. + * @param term The term to expand. Must not be {@code null} or blank. + * @param pos The part of speech to expand as. Must not be {@code null}. * @return The expansions, deduplicated and ordered by descending weight; empty when the term * (and its lemma, when a lemmatizer is configured) is not in the lexicon. - * @throws IllegalArgumentException Thrown if {@code term} is null or blank or {@code pos} is - * null. + * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or blank or + * {@code pos} is {@code null}. */ public List expand(String term, WordNetPOS pos) { if (pos == null) { - throw new IllegalArgumentException("The pos must not be null."); + throw new IllegalArgumentException("pos must not be null"); } return collect(term, List.of(pos)); } @@ -166,10 +166,10 @@ public List expand(String term, WordNetPOS pos) { /** * Expands a term across all parts of speech. * - * @param term The term to expand; must not be null or blank. + * @param term The term to expand. Must not be {@code null} or blank. * @return The expansions across every part of speech, deduplicated and ordered by descending * weight; empty when the term is not in the lexicon. - * @throws IllegalArgumentException Thrown if {@code term} is null or blank. + * @throws IllegalArgumentException Thrown if {@code term} is {@code null} or blank. */ public List expand(String term) { return collect(term, List.of(WordNetPOS.values())); @@ -186,7 +186,7 @@ public List expand(String term) { */ private List collect(String term, List poses) { if (term == null || StringUtil.isBlank(term)) { - throw new IllegalArgumentException("The term must not be null or blank."); + throw new IllegalArgumentException("term must not be null or blank"); } final Map best = new HashMap<>(); final Set excluded = new HashSet<>(); @@ -255,9 +255,7 @@ private String resolveSubject(String term, WordNetPOS pos) { private void expandSense(Synset sense, int rank, double senseWeight, Map best, Set excluded) { if (senseWeight == 0.0) { - // The decay product underflowed to zero in double arithmetic. A zero weight - // carries no ranking signal, so the sense and everything derived from it is - // dropped instead of emitted outside the documented (0, 1] weight range. + // Underflowed to zero: no ranking signal left, and zero is outside the documented range. return; } for (final String lemma : sense.lemmas()) { @@ -316,7 +314,7 @@ private void expandSense(Synset sense, int rank, double senseWeight, * @param synset The synset whose hypernyms are collected. Must not be {@code null}. * @return The hypernym synset ids in source order, direct relations first. */ - private static List hypernymsOf(Synset synset) { + private List hypernymsOf(Synset synset) { final List direct = synset.related(WordNetRelation.HYPERNYM); final List instance = synset.related(WordNetRelation.INSTANCE_HYPERNYM); if (instance.isEmpty()) { @@ -337,8 +335,8 @@ private static List hypernymsOf(Synset synset) { * @param excluded The folded terms that are never reported. * @param candidate The expansion to offer. Must not be {@code null}. */ - private static void offer(Map best, Set excluded, - Expansion candidate) { + private void offer(Map best, Set excluded, + Expansion candidate) { final String key = LemmaFolding.fold(candidate.term()); if (excluded.contains(key)) { return; @@ -365,11 +363,11 @@ public static final class Builder { * Creates a builder over the given lexicon; use {@link LexicalExpander#builder}. * * @param lexicon The knowledge base to expand against. Must not be {@code null}. - * @throws IllegalArgumentException Thrown if {@code lexicon} is null. + * @throws IllegalArgumentException Thrown if {@code lexicon} is {@code null}. */ private Builder(LexicalKnowledgeBase lexicon) { if (lexicon == null) { - throw new IllegalArgumentException("The lexicon must not be null."); + throw new IllegalArgumentException("lexicon must not be null"); } this.lexicon = lexicon; } @@ -378,13 +376,13 @@ private Builder(LexicalKnowledgeBase lexicon) { * Configures a lemmatizer used when the input term itself is not in the lexicon. It is * invoked with the {@link WordNetPOS} name as the tag. * - * @param lemmatizer The fallback lemmatizer; must not be null. + * @param lemmatizer The fallback lemmatizer. Must not be {@code null}. * @return This builder. - * @throws IllegalArgumentException Thrown if {@code lemmatizer} is null. + * @throws IllegalArgumentException Thrown if {@code lemmatizer} is {@code null}. */ public Builder lemmatizer(Lemmatizer lemmatizer) { if (lemmatizer == null) { - throw new IllegalArgumentException("The lemmatizer must not be null."); + throw new IllegalArgumentException("lemmatizer must not be null"); } this.lemmatizer = lemmatizer; return this; @@ -399,7 +397,7 @@ public Builder lemmatizer(Lemmatizer lemmatizer) { */ public Builder maxSenses(int maxSenses) { if (maxSenses < 1) { - throw new IllegalArgumentException("The maxSenses must be positive: " + maxSenses); + throw new IllegalArgumentException("maxSenses must be positive: " + maxSenses); } this.maxSenses = maxSenses; return this; @@ -415,7 +413,7 @@ public Builder maxSenses(int maxSenses) { public Builder hypernymDepth(int hypernymDepth) { if (hypernymDepth < 0) { throw new IllegalArgumentException( - "The hypernymDepth must not be negative: " + hypernymDepth); + "hypernymDepth must not be negative: " + hypernymDepth); } this.hypernymDepth = hypernymDepth; return this; @@ -442,7 +440,7 @@ public Builder includeHyponyms(boolean includeHyponyms) { public Builder maxExpansions(int maxExpansions) { if (maxExpansions < 1) { throw new IllegalArgumentException( - "The maxExpansions must be positive: " + maxExpansions); + "maxExpansions must be positive: " + maxExpansions); } this.maxExpansions = maxExpansions; return this; @@ -458,7 +456,7 @@ public Builder maxExpansions(int maxExpansions) { public Builder senseDecay(double senseDecay) { if (!(senseDecay > 0 && senseDecay <= 1)) { throw new IllegalArgumentException( - "The senseDecay must be in (0, 1]: " + senseDecay); + "senseDecay must be in (0, 1]: " + senseDecay); } this.senseDecay = senseDecay; return this; @@ -474,7 +472,7 @@ public Builder senseDecay(double senseDecay) { public Builder depthDecay(double depthDecay) { if (!(depthDecay > 0 && depthDecay <= 1)) { throw new IllegalArgumentException( - "The depthDecay must be in (0, 1]: " + depthDecay); + "depthDecay must be in (0, 1]: " + depthDecay); } this.depthDecay = depthDecay; return this; diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java index d0a28796fb..050f85a16c 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java @@ -129,6 +129,7 @@ public double wuPalmer(String synsetId, String otherSynsetId) { */ public double leacockChodorow(String synsetId, String otherSynsetId, int taxonomyDepth) { + validateIds(synsetId, otherSynsetId); if (taxonomyDepth <= 0) { throw new IllegalArgumentException( "taxonomyDepth must be positive: " + taxonomyDepth); @@ -184,7 +185,13 @@ private void validateIds(String synsetId, String otherSynsetId) { } } - /** Collects every ancestor with its minimal upward distance, the synset included. */ + /** + * Collects every ancestor with its minimal upward distance, the synset itself included at + * distance zero. + * + * @param synsetId The synset to walk up from. + * @return The upward distance to each reachable ancestor, keyed by synset identifier. + */ private Map depthsAbove(String synsetId) { final Map depths = new HashMap<>(); final Deque queue = new ArrayDeque<>(); @@ -203,7 +210,13 @@ private Map depthsAbove(String synsetId) { return depths; } - /** Measures a synset's depth from its taxonomy root, the shortest way up. */ + /** + * Measures a synset's depth as the distance to its farthest ancestor, which is the taxonomy + * root reached the long way round when several paths lead up. + * + * @param synsetId The synset to measure. + * @return The edge count to the farthest ancestor, {@code 0} for a root. + */ private int depthFromRoot(String synsetId) { final Map above = depthsAbove(synsetId); int deepest = 0; @@ -213,6 +226,12 @@ private int depthFromRoot(String synsetId) { return deepest; } + /** + * Collects the synsets one taxonomy edge above a synset. + * + * @param synsetId The synset whose parents are collected. + * @return The plain hypernyms followed by the instance hypernyms. + */ private Iterable hypernyms(String synsetId) { final List parents = new ArrayList<>( knowledgeBase.related(synsetId, WordNetRelation.HYPERNYM)); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java index 5c14a1c554..cba6828fc5 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/HypernymTyperTest.java @@ -24,8 +24,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import opennlp.tools.wordnet.WordNetRelation; - /** * Tests that {@link HypernymTyper} labels a word by its nearest anchored hypernym over * the fixture taxonomy of {@link SynsetSimilarityTest}, follows instance hypernymy, @@ -43,22 +41,10 @@ private static HypernymTyper typer() { } /** - * @return The shared fixture taxonomy. Never {@code null}. + * @return The taxonomy shared with {@link SynsetSimilarityTest}. Never {@code null}. */ private static SynsetSimilarityTest.FixtureKnowledgeBase taxonomy() { - final SynsetSimilarityTest.FixtureKnowledgeBase kb = - new SynsetSimilarityTest.FixtureKnowledgeBase(); - kb.add("n1", "entity", WordNetRelation.HYPERNYM); - kb.add("n2", "physical", WordNetRelation.HYPERNYM, "n1"); - kb.add("n3", "organism", WordNetRelation.HYPERNYM, "n2"); - kb.add("n4", "person", WordNetRelation.HYPERNYM, "n3"); - kb.add("n5", "scientist", WordNetRelation.HYPERNYM, "n4"); - kb.add("n6", "chemist", WordNetRelation.HYPERNYM, "n5"); - kb.add("n7", "location", WordNetRelation.HYPERNYM, "n2"); - kb.add("n8", "city", WordNetRelation.HYPERNYM, "n7"); - kb.add("n11", "paris", WordNetRelation.INSTANCE_HYPERNYM, "n8"); - kb.add("n12", "abstract", WordNetRelation.HYPERNYM); - return kb; + return SynsetSimilarityTest.taxonomy(); } /** diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java index c52b01da7d..176c1142d7 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderLexiconTest.java @@ -34,7 +34,7 @@ * feed the expander, and the Morphy lemmatizer bridges inflected input, exercising the whole * stack the way a consumer wires it. */ -class LexicalExpanderLexiconTest { +public class LexicalExpanderLexiconTest { @Test void testExpansionOverTheWnLmfLexicon() { diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java index 91da149935..85f8c700bb 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java @@ -20,8 +20,12 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import opennlp.tools.lemmatizer.Lemmatizer; import opennlp.tools.wordnet.LexicalKnowledgeBase; @@ -33,6 +37,7 @@ import static opennlp.wordnet.ExpansionAssertions.find; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -43,7 +48,7 @@ * ranking, hypernym depth and decay, hyponym opt-in, deduplication, exclusion of the input, * cycle termination, and configuration validation. */ -class LexicalExpanderTest { +public class LexicalExpanderTest { // dog: sense 1 = {dog, domestic dog} -> canid -> carnivore, with hyponym puppy; // sense 2 = {dog, frank, hot dog} -> sausage. The verb sense = {dog, chase}. @@ -140,7 +145,7 @@ void testUnderscoreInputReachesTheSpaceFoldedLexiconEntry() { void testTheInputTermIsNeverAnExpansion() { for (final Expansion expansion : LexicalExpander.builder(lexicon()).build().expand("dog", WordNetPOS.NOUN)) { - assertTrue(!expansion.term().equalsIgnoreCase("dog"), "got " + expansion); + assertFalse(expansion.term().equalsIgnoreCase("dog"), "got " + expansion); } } @@ -306,27 +311,39 @@ void testUnderflowedWeightsAreDropped() { } } + private static Stream invalidExpansions() { + return Stream.of( + Arguments.of(null, Kind.SYNONYM, 0, 0, 1.0), + Arguments.of(" ", Kind.SYNONYM, 0, 0, 1.0), + Arguments.of("dog", null, 0, 0, 1.0), + Arguments.of("dog", Kind.SYNONYM, -1, 0, 1.0), + Arguments.of("dog", Kind.SYNONYM, 0, -1, 1.0), + Arguments.of("dog", Kind.SYNONYM, 0, 0, 0.0), + Arguments.of("dog", Kind.SYNONYM, 0, 0, 1.5), + Arguments.of("dog", Kind.SYNONYM, 0, 0, Double.NaN)); + } + /** * Verifies that the {@link Expansion} record rejects every component * outside its documented range with a loud exception. */ + @ParameterizedTest + @MethodSource("invalidExpansions") + void testExpansionValidatesItsComponents(String term, Kind kind, int depth, int senseRank, + double weight) { + assertThrows(IllegalArgumentException.class, + () -> new Expansion(term, kind, depth, senseRank, weight)); + } + + /** Verifies that a fully valid component set is accepted. */ @Test - void testExpansionValidatesItsComponents() { - assertThrows(IllegalArgumentException.class, () -> new Expansion( - null, Kind.SYNONYM, 0, 0, 1.0)); - assertThrows(IllegalArgumentException.class, () -> new Expansion( - " ", Kind.SYNONYM, 0, 0, 1.0)); - assertThrows(IllegalArgumentException.class, () -> new Expansion( - "dog", null, 0, 0, 1.0)); - assertThrows(IllegalArgumentException.class, () -> new Expansion( - "dog", Kind.SYNONYM, -1, 0, 1.0)); - assertThrows(IllegalArgumentException.class, () -> new Expansion( - "dog", Kind.SYNONYM, 0, -1, 1.0)); - assertThrows(IllegalArgumentException.class, () -> new Expansion( - "dog", Kind.SYNONYM, 0, 0, 0.0)); - assertThrows(IllegalArgumentException.class, () -> new Expansion( - "dog", Kind.SYNONYM, 0, 0, 1.5)); - assertThrows(IllegalArgumentException.class, () -> new Expansion( - "dog", Kind.SYNONYM, 0, 0, Double.NaN)); + void testExpansionAcceptsValidComponents() { + final Expansion expansion = new Expansion("dog", Kind.HYPERNYM, 2, 1, 0.25); + + assertEquals("dog", expansion.term()); + assertEquals(Kind.HYPERNYM, expansion.kind()); + assertEquals(2, expansion.depth()); + assertEquals(1, expansion.senseRank()); + assertEquals(0.25, expansion.weight()); } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java index a379b7a0c3..c89a62499a 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java @@ -32,6 +32,8 @@ import opennlp.wordnet.LexicalExpander.Expansion; import opennlp.wordnet.LexicalExpander.Kind; +import static opennlp.wordnet.ExpansionAssertions.find; + /** * Runs the manual's lexical expansion and synset similarity examples (docbkx * {@code wordnet.xml}) verbatim: every value the chapter states is asserted here, so a @@ -113,15 +115,6 @@ private static void add(Map byId, String id, String lemma, Strin byId.put(id, new Synset(id, WordNetPOS.NOUN, List.of(lemma), "fixture", relations)); } - private static Expansion find(List expansions, String term) { - for (final Expansion expansion : expansions) { - if (term.equals(expansion.term())) { - return expansion; - } - } - return null; - } - /** * Default expansion of noun {@code dog}: synonym and depth-1 hypernym weights. */ diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java index 3264970585..48accbc230 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java @@ -17,6 +17,7 @@ package opennlp.wordnet; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -31,8 +32,8 @@ import opennlp.tools.wordnet.WordNetRelation; /** - * Tests the taxonomy measures and the hypernym typer against a project-authored - * miniature taxonomy; no external lexicon data is involved. + * Tests the taxonomy measures against a project-authored miniature taxonomy; no external + * lexicon data is involved. {@link HypernymTyperTest} shares the same taxonomy. */ public class SynsetSimilarityTest { @@ -47,7 +48,7 @@ void add(String id, String lemma, WordNetRelation relation, String... parents) { final Synset synset = new Synset(id, WordNetPOS.NOUN, List.of(lemma), "fixture", relations); byId.put(id, synset); - byLemma.computeIfAbsent(lemma, key -> new java.util.ArrayList<>()).add(synset); + byLemma.computeIfAbsent(lemma, key -> new ArrayList<>()).add(synset); } @Override @@ -61,7 +62,8 @@ public Optional synset(String synsetId) { } } - private static FixtureKnowledgeBase taxonomy() { + /** {@return the taxonomy both this test and {@link HypernymTyperTest} assert against} */ + static FixtureKnowledgeBase taxonomy() { final FixtureKnowledgeBase kb = new FixtureKnowledgeBase(); kb.add("n1", "entity", WordNetRelation.HYPERNYM); kb.add("n2", "physical", WordNetRelation.HYPERNYM, "n1"); @@ -115,37 +117,29 @@ void testInstanceHypernymsCountAsEdges() { Assertions.assertEquals(0.5, similarity.path("n11", "n8"), 1e-9); } - @Test - void testTyperFindsTheNearestAnchor() { - final HypernymTyper typer = new HypernymTyper(taxonomy(), Map.of( - "person", "person", "location", "location", "organization", "organization")); - Assertions.assertEquals("person", typer.type("chemist").orElseThrow()); - Assertions.assertEquals("location", typer.type("city").orElseThrow()); - Assertions.assertEquals("organization", typer.type("company").orElseThrow()); - Assertions.assertEquals("location", typer.typeSynset("n11").orElseThrow()); - Assertions.assertTrue(typer.type("entity").isEmpty()); - Assertions.assertTrue(typer.type("blorp").isEmpty()); - } - - @Test - void testMoreSpecificAnchorsWin() { - final HypernymTyper typer = new HypernymTyper(taxonomy(), Map.of( - "person", "person", "scientist", "researcher")); - Assertions.assertEquals("researcher", typer.type("chemist").orElseThrow()); - Assertions.assertEquals("person", typer.type("person").orElseThrow()); - } - @Test void testInvalidArguments() { Assertions.assertThrows(IllegalArgumentException.class, () -> new SynsetSimilarity(null)); + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); Assertions.assertThrows(IllegalArgumentException.class, - () -> new SynsetSimilarity(taxonomy()).path(null, "n1")); + () -> similarity.path(null, "n1")); Assertions.assertThrows(IllegalArgumentException.class, - () -> new HypernymTyper(taxonomy(), Map.of())); + () -> similarity.path("n1", null)); Assertions.assertThrows(IllegalArgumentException.class, - () -> new HypernymTyper(taxonomy(), Map.of("blorp", "thing"))); - final HypernymTyper typer = new HypernymTyper(taxonomy(), Map.of("person", "person")); - Assertions.assertThrows(IllegalArgumentException.class, () -> typer.type(" ")); + () -> similarity.wuPalmer(null, "n1")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> similarity.shortestDistance("n1", null)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> similarity.leacockChodorow(null, "n1", 10)); + } + + @Test + void testUnknownSynsetsAreUnrelatedRatherThanFatal() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + Assertions.assertEquals(-1, similarity.shortestDistance("n5", "missing")); + Assertions.assertEquals(0.0, similarity.path("n5", "missing"), 1e-9); + Assertions.assertEquals(0.0, similarity.wuPalmer("n5", "missing"), 1e-9); + Assertions.assertEquals(0.0, similarity.leacockChodorow("n5", "missing", 10), 1e-9); } } From 3dddfed896e169d815fd6408c68772a5dc864be2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:38:23 -0400 Subject: [PATCH 12/15] OPENNLP-1887: Make the manual's taxonomies explicit and pin boundary and default behavior Show the expansion and similarity taxonomies in the chapter's listing comments so every synset id and word in the listings exists in the shown lexicon, add a hypernym-anchored typing section, and extend the mirror test to build exactly those taxonomies and assert every printed value. Pin the formula-driven numeric regimes (Wu-Palmer zero at the root pair, negative Leacock-Chodorow under an understated depth budget) and the expander builder's accepting boundaries and defaults (senseDecay 1.0, hypernymDepth 0, maxSenses 3, maxExpansions 20) against fixture lemmas that exceed the defaults. --- opennlp-docs/src/docbkx/wordnet.xml | 53 ++++++++++++--- .../opennlp/wordnet/LexicalExpanderTest.java | 68 ++++++++++++++++++- .../LexicalExpansionUsageExampleTest.java | 59 +++++++++++----- .../opennlp/wordnet/SynsetSimilarityTest.java | 35 ++++++++++ 4 files changed, 187 insertions(+), 28 deletions(-) diff --git a/opennlp-docs/src/docbkx/wordnet.xml b/opennlp-docs/src/docbkx/wordnet.xml index 82d5e910ca..3e2d29b567 100644 --- a/opennlp-docs/src/docbkx/wordnet.xml +++ b/opennlp-docs/src/docbkx/wordnet.xml @@ -27,7 +27,8 @@ WndbReader for the classic Princeton WordNet database file layout. Both return an immutable, thread-safe knowledge base. On top of lookup, the module can Morphy-lemmatize, expand a term through synonym and - hypernym links, and score synset similarity on the hypernym graph. + hypernym links, score synset similarity on the hypernym graph, and type + nouns by their nearest anchored hypernym. @@ -117,10 +118,14 @@ lemmatizer.lemmatize(new String[] {"dogs"}, new String[] {"NNS"})[0]; // "dog"] senses multiply by the sense decay, and each hypernym or hyponym step multiplies by the depth decay. The input term itself is never returned. Defaults use depth 1, sense decay 0.5, and - depth decay 0.5. - LexicalExpansionUsageExampleTest asserts the behavior shown - here. + depth decay 0.5. The example runs against the two-sense + miniature taxonomy in the leading comment, not the lookup fixture above: + the mini fixture carries only one sense of dog, so it cannot + show the sense decay. LexicalExpansionUsageExampleTest + builds exactly that taxonomy and asserts the behavior shown here. expansions = LexicalExpander.builder(lexicon) .build() .expand("dog", WordNetPOS.NOUN); @@ -139,14 +144,44 @@ List expansions = LexicalExpander.builder(lexicon) hypernym graph. Path similarity is 1 / (1 + d) for the shortest distance d through a common ancestor. Wu-Palmer similarity relates the depth of the deepest common ancestor to the depths - of both synsets. Unrelated synsets score 0. - LexicalExpansionUsageExampleTest asserts the behavior shown - here. + of both synsets. Unrelated synsets score 0. The example runs + against the miniature taxonomy in the leading comment, with synset ids + n1 to n9; + LexicalExpansionUsageExampleTest builds exactly that + taxonomy and asserts the behavior shown here. physical (n2) > organism (n3) > person (n4) +// > scientist (n5) > chemist (n6) +// physical (n2) > location (n7) > city (n8) +// paris (n9) is an instance of city SynsetSimilarity similarity = new SynsetSimilarity(lexicon); -similarity.path("n6", "n5"); // 0.5 (chemist to scientist) -similarity.wuPalmer("n5", "n6"); // 8.0/9.0]]> +similarity.path("n6", "n5"); // 0.5 (chemist to scientist) +similarity.wuPalmer("n5", "n6"); // 8.0/9.0 (deep shared ancestry) +similarity.path("n6", "n8"); // 1.0/7.0 (chemist to city, six edges)]]> + + + + +
+ Hypernym-anchored typing + + HypernymTyper types a noun by walking its hypernym chain to + the nearest registered anchor: the caller maps anchor lemmas to the + labels they confer, and any noun whose senses lead up to an anchor + receives that anchor's label. Instance hypernyms count, so named + entities reach their class anchors. The walk is upward only, and a noun + reaching no anchor gets no type. The example runs against the taxonomy + of the previous section; HypernymTyperTest and + LexicalExpansionUsageExampleTest assert the behavior shown + here. +
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java index 85f8c700bb..67127a0f22 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpanderTest.java @@ -16,8 +16,10 @@ */ package opennlp.wordnet; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Stream; @@ -54,6 +56,8 @@ public class LexicalExpanderTest { // sense 2 = {dog, frank, hot dog} -> sausage. The verb sense = {dog, chase}. // hot dog: the standalone multiword sense {hot dog, red hot}. // alpha <-> beta form a malformed hypernym cycle. + // poly: four singleton-synonym senses, one more than the default maxSenses of 3. + // hub: one synset with 24 synonyms, more than the default maxExpansions of 20. // Lookups fold through LemmaFolding, exactly as the readers fold their keys at load time. private static LexicalKnowledgeBase lexicon() { final Map synsets = new HashMap<>(); @@ -79,8 +83,23 @@ private static LexicalKnowledgeBase lexicon() { Map.of(WordNetRelation.HYPERNYM, List.of("c2"))); final Synset c2 = new Synset("c2", WordNetPOS.NOUN, List.of("beta"), "cycle end", Map.of(WordNetRelation.HYPERNYM, List.of("c1"))); + final Synset p1 = new Synset("p1", WordNetPOS.NOUN, List.of("poly", "poly-one"), + "first sense", Map.of()); + final Synset p2 = new Synset("p2", WordNetPOS.NOUN, List.of("poly", "poly-two"), + "second sense", Map.of()); + final Synset p3 = new Synset("p3", WordNetPOS.NOUN, List.of("poly", "poly-three"), + "third sense", Map.of()); + final Synset p4 = new Synset("p4", WordNetPOS.NOUN, List.of("poly", "poly-four"), + "fourth sense", Map.of()); + final List hubMembers = new ArrayList<>(); + hubMembers.add("hub"); + for (int i = 1; i <= 24; i++) { + hubMembers.add(String.format(Locale.ROOT, "spoke%02d", i)); + } + final Synset b1 = new Synset("b1", WordNetPOS.NOUN, hubMembers, "wide synset", Map.of()); - for (final Synset synset : List.of(n1, n2, n3, n4, n5, n6, v1, m1, c1, c2)) { + for (final Synset synset : List.of(n1, n2, n3, n4, n5, n6, v1, m1, c1, c2, + p1, p2, p3, p4, b1)) { synsets.put(synset.id(), synset); } senses.put("dog|NOUN", List.of(n1, n5)); @@ -88,6 +107,8 @@ private static LexicalKnowledgeBase lexicon() { senses.put("domestic dog|NOUN", List.of(n1)); senses.put("hot dog|NOUN", List.of(m1)); senses.put("alpha|NOUN", List.of(c1)); + senses.put("poly|NOUN", List.of(p1, p2, p3, p4)); + senses.put("hub|NOUN", List.of(b1)); return new LexicalKnowledgeBase() { @Override @@ -270,6 +291,51 @@ public List> lemmatize(List tokens, List tags) { assertEquals(List.of(), expander.expand("cats", WordNetPOS.NOUN)); } + /** + * Verifies the accepting side of the builder boundaries: {@code senseDecay(1.0)} is the + * closed upper bound of {@code (0, 1]} and keeps later senses at full weight, and + * {@code hypernymDepth(0)} is the accepted lower bound and disables hypernym expansion + * without touching synonyms. + */ + @Test + void testBoundaryConfigurationsAreAccepted() { + final List undecayed = LexicalExpander.builder(lexicon()) + .senseDecay(1.0).build().expand("dog", WordNetPOS.NOUN); + assertEquals(1.0, find(undecayed, "frank").weight()); + assertEquals(1, find(undecayed, "frank").senseRank()); + + final List synonymsOnly = LexicalExpander.builder(lexicon()) + .hypernymDepth(0).build().expand("dog", WordNetPOS.NOUN); + assertNull(find(synonymsOnly, "canid")); + assertNotNull(find(synonymsOnly, "domestic dog")); + } + + /** + * Pins the default {@code maxSenses} of {@code 3} against a lemma with four senses: the + * third sense still contributes, the fourth never does. + */ + @Test + void testDefaultMaxSensesIsThree() { + final List expansions = + LexicalExpander.builder(lexicon()).build().expand("poly", WordNetPOS.NOUN); + + assertEquals(1.0, find(expansions, "poly-one").weight()); + assertEquals(0.25, find(expansions, "poly-three").weight()); + assertNull(find(expansions, "poly-four")); + } + + /** + * Pins the default {@code maxExpansions} of {@code 20} against a synset with 24 synonyms: + * the default build caps the result at 20, and raising the cap shows all 24 were available. + */ + @Test + void testDefaultMaxExpansionsIsTwenty() { + assertEquals(20, + LexicalExpander.builder(lexicon()).build().expand("hub", WordNetPOS.NOUN).size()); + assertEquals(24, LexicalExpander.builder(lexicon()).maxExpansions(30).build() + .expand("hub", WordNetPOS.NOUN).size()); + } + @Test void testValidationFailsLoudly() { assertThrows(IllegalArgumentException.class, () -> LexicalExpander.builder(null)); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java index c89a62499a..7edb1ad886 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java @@ -17,6 +17,7 @@ package opennlp.wordnet; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -35,10 +36,10 @@ import static opennlp.wordnet.ExpansionAssertions.find; /** - * Runs the manual's lexical expansion and synset similarity examples (docbkx - * {@code wordnet.xml}) verbatim: every value the chapter states is asserted here, so a - * change breaking this test breaks the manual. The taxonomy is a hand-built miniature - * matching the shapes used elsewhere in this module's tests. + * Runs the manual's lexical expansion, synset similarity, and hypernym-anchored typing + * examples (docbkx {@code wordnet.xml}) verbatim: every value the chapter states is asserted + * here, so a change breaking this test breaks the manual. The taxonomies are the hand-built + * miniatures the chapter shows in its listing comments. */ public class LexicalExpansionUsageExampleTest { @@ -83,23 +84,26 @@ public Optional synset(String synsetId) { } /** + * The taxonomy shown in the chapter's similarity and typing listings: * chemist -> scientist -> person -> organism -> physical -> entity; city -> location -> - * physical. + * physical; paris is an instance of city. */ private static LexicalKnowledgeBase similarityTaxonomy() { final Map byId = new HashMap<>(); - add(byId, "n1", "entity"); - add(byId, "n2", "physical", "n1"); - add(byId, "n3", "organism", "n2"); - add(byId, "n4", "person", "n3"); - add(byId, "n5", "scientist", "n4"); - add(byId, "n6", "chemist", "n5"); - add(byId, "n7", "location", "n2"); - add(byId, "n8", "city", "n7"); + final Map> byLemma = new HashMap<>(); + add(byId, byLemma, "n1", "entity", WordNetRelation.HYPERNYM); + add(byId, byLemma, "n2", "physical", WordNetRelation.HYPERNYM, "n1"); + add(byId, byLemma, "n3", "organism", WordNetRelation.HYPERNYM, "n2"); + add(byId, byLemma, "n4", "person", WordNetRelation.HYPERNYM, "n3"); + add(byId, byLemma, "n5", "scientist", WordNetRelation.HYPERNYM, "n4"); + add(byId, byLemma, "n6", "chemist", WordNetRelation.HYPERNYM, "n5"); + add(byId, byLemma, "n7", "location", WordNetRelation.HYPERNYM, "n2"); + add(byId, byLemma, "n8", "city", WordNetRelation.HYPERNYM, "n7"); + add(byId, byLemma, "n9", "paris", WordNetRelation.INSTANCE_HYPERNYM, "n8"); return new LexicalKnowledgeBase() { @Override public List lookup(String lemma, WordNetPOS pos) { - return List.of(); + return byLemma.getOrDefault(lemma, List.of()); } @Override @@ -109,10 +113,13 @@ public Optional synset(String synsetId) { }; } - private static void add(Map byId, String id, String lemma, String... parents) { + private static void add(Map byId, Map> byLemma, + String id, String lemma, WordNetRelation relation, String... parents) { final Map> relations = parents.length == 0 - ? Map.of() : Map.of(WordNetRelation.HYPERNYM, List.of(parents)); - byId.put(id, new Synset(id, WordNetPOS.NOUN, List.of(lemma), "fixture", relations)); + ? Map.of() : Map.of(relation, List.of(parents)); + final Synset synset = new Synset(id, WordNetPOS.NOUN, List.of(lemma), "fixture", relations); + byId.put(id, synset); + byLemma.computeIfAbsent(lemma, key -> new ArrayList<>()).add(synset); } /** @@ -143,12 +150,28 @@ void testExpandDogNoun() { } /** - * Path and Wu-Palmer scores on the miniature scientist/city taxonomy. + * Path and Wu-Palmer scores on the miniature scientist/city taxonomy, exactly as the + * chapter's similarity listing prints them. */ @Test void testSynsetSimilarityScores() { final SynsetSimilarity similarity = new SynsetSimilarity(similarityTaxonomy()); Assertions.assertEquals(0.5, similarity.path("n6", "n5"), 1e-9); Assertions.assertEquals(8.0 / 9.0, similarity.wuPalmer("n5", "n6"), 1e-9); + Assertions.assertEquals(1.0 / 7.0, similarity.path("n6", "n8"), 1e-9); + } + + /** + * Hypernym-anchored typing over the same taxonomy, exactly as the chapter's typing + * listing prints it: a chemist is a person, paris reaches location through instance + * hypernymy, and an ancestor of an anchor is never typed because the walk is upward only. + */ + @Test + void testHypernymAnchoredTyping() { + final HypernymTyper typer = new HypernymTyper(similarityTaxonomy(), + Map.of("person", "person", "location", "location")); + Assertions.assertEquals(Optional.of("person"), typer.type("chemist")); + Assertions.assertEquals(Optional.of("location"), typer.type("paris")); + Assertions.assertEquals(Optional.empty(), typer.type("organism")); } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java index 48accbc230..214fc4bb6b 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java @@ -101,6 +101,41 @@ void testWuPalmerRewardsDeepSharedAncestry() { Assertions.assertEquals(0.0, similarity.wuPalmer("n6", "n12"), 1e-9); } + /** + * Pins the self-similarity regime at the taxonomy root. Wu-Palmer counts depth in + * edges from the root, so the root itself has depth zero and the formula + * {@code 2 * depth(lcs) / (depth(a) + depth(b))} has a zero denominator for the pair + * (root, root); that case is skipped and the score is {@code 0.0}, while path + * similarity of any synset with itself, root included, is {@code 1.0} by + * {@code 1 / (1 + 0)}. Both values follow the documented formulas; this test keeps + * the asymmetry a deliberate pin rather than a surprise. + */ + @Test + void testRootSelfSimilarityFollowsTheFormulas() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + Assertions.assertEquals(1.0, similarity.path("n1", "n1"), 1e-9); + Assertions.assertEquals(0.0, similarity.wuPalmer("n1", "n1"), 1e-9); + // Away from the root the same formula does yield full self-similarity. + Assertions.assertEquals(1.0, similarity.wuPalmer("n5", "n5"), 1e-9); + } + + /** + * Pins the sign regime of Leacock-Chodorow. The measure is + * {@code -log((d + 1) / (2 * taxonomyDepth))} with a caller-supplied depth; when the + * shortest path exceeds the stated depth budget, {@code d + 1 > 2 * taxonomyDepth}, + * the ratio exceeds one and the score goes negative rather than clamping at zero. + * That is the documented formula behavior for an understated taxonomy depth, pinned + * here so the negative range is a deliberate contract. + */ + @Test + void testLeacockChodorowGoesNegativeWhenDistanceExceedsTheDepthBudget() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + // chemist to city is six edges, so (6 + 1) / (2 * 3) is greater than one + final double score = similarity.leacockChodorow("n6", "n8", 3); + Assertions.assertEquals(-Math.log(7.0 / 6.0), score, 1e-9); + Assertions.assertTrue(score < 0.0); + } + @Test void testLeacockChodorow() { final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); From fc89463e5e68ebb28785d84572515de90746caa3 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 08:31:38 -0400 Subject: [PATCH 13/15] OPENNLP-1887: Pin that Wu-Palmer must separate root-only ancestry from no ancestry city (n8) and company (n10) share only the taxonomy root, yet wuPalmer returns 0.0 for them, the same score as the disconnected chemist/abstract pair. The documented contract reserves 0 for pairs with no shared ancestor. Red: testWuPalmerDistinguishesRootOnlyAncestryFromNoAncestry fails with 'city and company share entity; expected positive Wu-Palmer, got 0.0'. --- .../opennlp/wordnet/SynsetSimilarityTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java index 214fc4bb6b..ac7e2742ec 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java @@ -101,6 +101,23 @@ void testWuPalmerRewardsDeepSharedAncestry() { Assertions.assertEquals(0.0, similarity.wuPalmer("n6", "n12"), 1e-9); } + /** + * Verifies that Wu-Palmer distinguishes synsets that share only the taxonomy root from + * synsets that share no ancestor at all. The documented return is {@code 0} only when + * the synsets share no ancestor; city ({@code n8}) and company ({@code n10}) both sit + * under entity ({@code n1}), so their score must be positive and distinct from the + * disconnected chemist/abstract pair. + */ + @Test + void testWuPalmerDistinguishesRootOnlyAncestryFromNoAncestry() { + final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); + final double rootOnly = similarity.wuPalmer("n8", "n10"); + Assertions.assertTrue(rootOnly > 0.0, + "city and company share entity; expected positive Wu-Palmer, got " + rootOnly); + Assertions.assertEquals(0.0, similarity.wuPalmer("n6", "n12"), 1e-9); + Assertions.assertNotEquals(similarity.wuPalmer("n6", "n12"), rootOnly); + } + /** * Pins the self-similarity regime at the taxonomy root. Wu-Palmer counts depth in * edges from the root, so the root itself has depth zero and the formula From fa0d25354ce06b612a98588d0e33932b739cf2c9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 08:33:06 -0400 Subject: [PATCH 14/15] OPENNLP-1887: Count Wu-Palmer depth in nodes so root-only ancestry scores positive The edge-counted formula gave the taxonomy root depth zero, so pairs whose only shared ancestor is the root collapsed to 0.0 and were indistinguishable from pairs sharing no ancestor, violating the documented contract that 0 is reserved for no shared ancestor. Wu and Palmer's node counting places the root at depth one; the formula 2 * depth(lcs) / (depth(a) + depth(b)) keeps its shape, the zero denominator guard becomes unreachable, and (root, root) self-similarity rises from the pinned 0.0 to a full 1.0. Under the corrected convention wuPalmer(n5, n6) moves from 8/9 to 10/11; the pinned tests, the usage example mirror test, and the manual's similarity listing and prose are updated to match. --- opennlp-docs/src/docbkx/wordnet.xml | 6 ++++-- .../opennlp/wordnet/SynsetSimilarity.java | 19 ++++++++++--------- .../LexicalExpansionUsageExampleTest.java | 2 +- .../opennlp/wordnet/SynsetSimilarityTest.java | 19 +++++++++---------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/opennlp-docs/src/docbkx/wordnet.xml b/opennlp-docs/src/docbkx/wordnet.xml index 3e2d29b567..895926408a 100644 --- a/opennlp-docs/src/docbkx/wordnet.xml +++ b/opennlp-docs/src/docbkx/wordnet.xml @@ -144,7 +144,9 @@ List expansions = LexicalExpander.builder(lexicon) hypernym graph. Path similarity is 1 / (1 + d) for the shortest distance d through a common ancestor. Wu-Palmer similarity relates the depth of the deepest common ancestor to the depths - of both synsets. Unrelated synsets score 0. The example runs + of both synsets, with depths counted in nodes so the root sits at depth + one and any shared ancestor scores above 0. Only unrelated + synsets score 0. The example runs against the miniature taxonomy in the leading comment, with synset ids n1 to n9; LexicalExpansionUsageExampleTest builds exactly that @@ -157,7 +159,7 @@ List expansions = LexicalExpander.builder(lexicon) SynsetSimilarity similarity = new SynsetSimilarity(lexicon); similarity.path("n6", "n5"); // 0.5 (chemist to scientist) -similarity.wuPalmer("n5", "n6"); // 8.0/9.0 (deep shared ancestry) +similarity.wuPalmer("n5", "n6"); // 10.0/11.0 (deep shared ancestry) similarity.path("n6", "n8"); // 1.0/7.0 (chemist to city, six edges)]]> diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java index 050f85a16c..1c51569e4a 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/SynsetSimilarity.java @@ -80,8 +80,10 @@ public double path(String synsetId, String otherSynsetId) { /** * Computes Wu-Palmer similarity: {@code 2 * depth(lcs) / (depth(a) + depth(b))}, - * with depths counted from the taxonomy root and the deepest common ancestor as the - * lcs. + * with depths counted in nodes from the taxonomy root, the root itself at depth one, + * and the deepest common ancestor as the lcs. Node counting keeps the score positive + * whenever any ancestor is shared, the root included, so {@code 0} is reserved for + * synsets that share no ancestor at all. * * @param synsetId The first synset identifier. Must not be {@code null}. * @param otherSynsetId The second synset identifier. Must not be {@code null}. @@ -100,13 +102,12 @@ public double wuPalmer(String synsetId, String otherSynsetId) { if (otherDistance == null) { continue; } - final int rootDepth = depthFromRoot(common.getKey()); - final int depthA = rootDepth + common.getValue(); - final int depthB = rootDepth + otherDistance; - if (depthA + depthB == 0) { - continue; - } - final double score = 2.0 * rootDepth / (depthA + depthB); + // Node counting: the root sits at depth one, so a shared ancestor always + // contributes a positive numerator and the denominator is never zero. + final int lcsDepth = depthFromRoot(common.getKey()) + 1; + final int depthA = lcsDepth + common.getValue(); + final int depthB = lcsDepth + otherDistance; + final double score = 2.0 * lcsDepth / (depthA + depthB); best = Math.max(best, score); } return best; diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java index 7edb1ad886..6fd8ea4a68 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexicalExpansionUsageExampleTest.java @@ -157,7 +157,7 @@ void testExpandDogNoun() { void testSynsetSimilarityScores() { final SynsetSimilarity similarity = new SynsetSimilarity(similarityTaxonomy()); Assertions.assertEquals(0.5, similarity.path("n6", "n5"), 1e-9); - Assertions.assertEquals(8.0 / 9.0, similarity.wuPalmer("n5", "n6"), 1e-9); + Assertions.assertEquals(10.0 / 11.0, similarity.wuPalmer("n5", "n6"), 1e-9); Assertions.assertEquals(1.0 / 7.0, similarity.path("n6", "n8"), 1e-9); } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java index ac7e2742ec..ac7aeaa50a 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/SynsetSimilarityTest.java @@ -93,8 +93,8 @@ void testPathSimilarity() { @Test void testWuPalmerRewardsDeepSharedAncestry() { final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); - // scientist and chemist share scientist itself at depth four - Assertions.assertEquals(8.0 / 9.0, similarity.wuPalmer("n5", "n6"), 1e-9); + // scientist and chemist share scientist itself, at node depth five from entity + Assertions.assertEquals(10.0 / 11.0, similarity.wuPalmer("n5", "n6"), 1e-9); final double siblingBranches = similarity.wuPalmer("n6", "n8"); Assertions.assertTrue(siblingBranches < similarity.wuPalmer("n5", "n6")); Assertions.assertTrue(siblingBranches > 0.0); @@ -120,19 +120,18 @@ void testWuPalmerDistinguishesRootOnlyAncestryFromNoAncestry() { /** * Pins the self-similarity regime at the taxonomy root. Wu-Palmer counts depth in - * edges from the root, so the root itself has depth zero and the formula - * {@code 2 * depth(lcs) / (depth(a) + depth(b))} has a zero denominator for the pair - * (root, root); that case is skipped and the score is {@code 0.0}, while path - * similarity of any synset with itself, root included, is {@code 1.0} by - * {@code 1 / (1 + 0)}. Both values follow the documented formulas; this test keeps - * the asymmetry a deliberate pin rather than a surprise. + * nodes from the root, so the root itself has depth one and the formula + * {@code 2 * depth(lcs) / (depth(a) + depth(b))} yields {@code 2 * 1 / (1 + 1)}, + * a full {@code 1.0}, for the pair (root, root), matching path similarity's + * {@code 1 / (1 + 0)}. Self-similarity is {@code 1.0} everywhere, root included; + * this test keeps that a deliberate pin rather than a surprise. */ @Test void testRootSelfSimilarityFollowsTheFormulas() { final SynsetSimilarity similarity = new SynsetSimilarity(taxonomy()); Assertions.assertEquals(1.0, similarity.path("n1", "n1"), 1e-9); - Assertions.assertEquals(0.0, similarity.wuPalmer("n1", "n1"), 1e-9); - // Away from the root the same formula does yield full self-similarity. + Assertions.assertEquals(1.0, similarity.wuPalmer("n1", "n1"), 1e-9); + // Away from the root the same formula also yields full self-similarity. Assertions.assertEquals(1.0, similarity.wuPalmer("n5", "n5"), 1e-9); } From a68762fe35d98562c701502b2ebd7dd8ba9f5420 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 16 Aug 2026 06:33:04 -0400 Subject: [PATCH 15/15] OPENNLP-1880: Fold WordNet lemmas with StringUtil.toLowerCase The canonical lemma fold used String.toLowerCase(Locale.ROOT), which applies the SpecialCasing one-to-many mappings. Fold with StringUtil.toLowerCase instead so the sense-index keys, the Morphy exception keys, and the query path all share the locale-independent one-to-one mapping. This branch stacks on the lexical knowledge base work, so the change belongs to that commit range and can be squashed into it on the next rebase. --- .../src/main/java/opennlp/wordnet/LemmaFolding.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java index 697246731f..259e6a12ad 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java @@ -18,7 +18,8 @@ import java.util.ArrayList; import java.util.List; -import java.util.Locale; + +import opennlp.tools.util.StringUtil; /** * The single home of the lemma fold and the space-separated field split this package relies on. @@ -32,8 +33,9 @@ private LemmaFolding() { } /** - * Folds a written form into its canonical shape: lowercase with the root locale, with the - * underscore some formats store in multiword lemmas treated as a space. + * Folds a written form into its canonical shape: lowercase with the locale-independent + * one-to-one mapping of {@link StringUtil#toLowerCase(CharSequence)}, with the underscore + * some formats store in multiword lemmas treated as a space. * * @param writtenForm The form as written in a source file or query. Must not be {@code null}. * @return The folded form. @@ -43,7 +45,7 @@ static String fold(String writtenForm) { if (writtenForm == null) { throw new IllegalArgumentException("WrittenForm must not be null"); } - return writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT); + return StringUtil.toLowerCase(writtenForm.replace('_', ' ')); } /**