Length-changing folds move offsets: collapsing a run of whitespace, folding a supplementary - * dash to one ASCII hyphen, or stripping invisible controls all shift every later character. A rung + * dash to one ASCII hyphen, or stripping invisible controls all shift every later character. A normalizer * that performs such a fold over the cursor-based {@link CharClass} engine can record those edits - * and expose them through {@link #normalizeAligned(CharSequence)}. A rung that delegates to + * and expose them through {@link #normalizeAligned(CharSequence)}. A normalizer that delegates to * {@link java.text.Normalizer} (NFC/NFKC) or to a stemmer cannot report its edits, so it does not * implement this interface; that is a deliberate capability split rather than an oversight.
* *{@code TextNormalizer.Builder.buildAligned()} composes a chain of these into a single * offset-aware pipeline whose {@link AlignedText} maps a match all the way back to the original * input. An interface-typed caller tests for the capability - * ({@code normalizer instanceof OffsetAwareNormalizer}) instead of depending on a concrete rung, + * ({@code normalizer instanceof OffsetAwareNormalizer}) instead of depending on a concrete normalizer, * the same plain {@code instanceof} pattern used by * {@code OffsetMappingNameFinder} (in the DL layer) rather than reflection.
*/ diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/InferenceOptions.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/InferenceOptions.java index b59aac226b..857fc12785 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/InferenceOptions.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/InferenceOptions.java @@ -88,7 +88,7 @@ public boolean isNormalizeWhitespace() { * produces still align with the input. Off by default. * *This is a one-for-one replacement, not the collapse-and-trim whitespace fold of the runtime - * {@code TextNormalizer.whitespace()} rung: runs of whitespace are not merged and leading or + * {@code TextNormalizer.whitespace()} normalizer: runs of whitespace are not merged and leading or * trailing whitespace is not removed, so offsets are preserved.
* * @param normalizeWhitespace Whether to normalize whitespace. diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java index 56a46b556d..a703e95575 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizer.java @@ -27,19 +27,36 @@ public class AggregateCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = 5514902020184083235L; private final CharSequenceNormalizer[] normalizers; - public AggregateCharSequenceNormalizer(CharSequenceNormalizer ... normalizers) { - this.normalizers = normalizers; + /** + * Creates an aggregate that applies the given normalizers in order. + * + * @param normalizers The normalizers to apply, first to last. Must not be + * {@code null} and must not contain {@code null}. The array is + * copied, so later changes to it do not reach this instance. + * @throws IllegalArgumentException Thrown if {@code normalizers} is {@code null} or + * contains {@code null}. + */ + public AggregateCharSequenceNormalizer(CharSequenceNormalizer... normalizers) { + if (normalizers == null) { + throw new IllegalArgumentException("The normalizers must not be null."); + } + for (CharSequenceNormalizer normalizer : normalizers) { + if (normalizer == null) { + throw new IllegalArgumentException("The normalizers must not contain null."); + } + } + this.normalizers = normalizers.clone(); } /** {@inheritDoc} */ @Override - public CharSequence normalize (CharSequence text) { + public CharSequence normalize(CharSequence text) { if (text == null) { throw new IllegalArgumentException("The text must not be null."); } - for (CharSequenceNormalizer normalizers : normalizers) { - text = normalizers.normalize(text); + for (CharSequenceNormalizer normalizer : normalizers) { + text = normalizer.normalize(text); } return text; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java index efa318f476..1f7a8c14f5 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/AlignedAggregateCharSequenceNormalizer.java @@ -17,11 +17,11 @@ package opennlp.tools.util.normalizer; /** - * An {@link OffsetAwareNormalizer} that applies a chain of offset-aware rungs in order and composes + * An {@link OffsetAwareNormalizer} that applies a chain of offset-aware normalizers in order and composes * their per-stage {@link Alignment}s with {@link Alignment#andThen(Alignment)}, so the result maps a * span found in the fully normalized text back to the original input through every stage. * - *Produced by {@code TextNormalizer.Builder.buildAligned()}, which validates that every rung is + *
Produced by {@code TextNormalizer.Builder.buildAligned()}, which validates that every normalizer is * offset-aware before constructing this.
*/ final class AlignedAggregateCharSequenceNormalizer implements OffsetAwareNormalizer { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizerTest.java new file mode 100644 index 0000000000..bb5a0267c2 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AggregateCharSequenceNormalizerTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.tools.util.normalizer; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class AggregateCharSequenceNormalizerTest { + + @Test + void testAppliesNormalizersInConstructionOrder() { + CharSequenceNormalizer upper = text -> text.toString().replace('a', 'B'); + CharSequenceNormalizer strip = text -> text.toString().replace("B", ""); + assertEquals("cc", new AggregateCharSequenceNormalizer(upper, strip) + .normalize("accB").toString()); + assertEquals("Bcc", new AggregateCharSequenceNormalizer(strip, upper) + .normalize("accB").toString()); + } + + @Test + void testRejectsNullNormalizersLoudly() { + IllegalArgumentException nullArray = assertThrows(IllegalArgumentException.class, + () -> new AggregateCharSequenceNormalizer((CharSequenceNormalizer[]) null)); + assertEquals("The normalizers must not be null.", nullArray.getMessage()); + + IllegalArgumentException nullElement = assertThrows(IllegalArgumentException.class, + () -> new AggregateCharSequenceNormalizer( + NfcCharSequenceNormalizer.getInstance(), null)); + assertEquals("The normalizers must not contain null.", nullElement.getMessage()); + } + + @Test + void testChangingTheCallerArrayDoesNotReachTheAggregate() { + CharSequenceNormalizer[] normalizers = {NfcCharSequenceNormalizer.getInstance()}; + AggregateCharSequenceNormalizer aggregate = + new AggregateCharSequenceNormalizer(normalizers); + normalizers[0] = text -> "changed"; + assertEquals("abc", aggregate.normalize("abc").toString()); + } + + @Test + void testRejectsNullTextLoudly() { + AggregateCharSequenceNormalizer aggregate = new AggregateCharSequenceNormalizer(); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> aggregate.normalize(null)); + assertEquals("The text must not be null.", e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java index 07098dfbc8..48eedc0332 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/AlignedNormalizerPipelineTest.java @@ -28,9 +28,9 @@ /** * Exercises {@link OffsetAwareNormalizer} and {@code TextNormalizer.Builder.buildAligned()}: the - * cursor-based rungs report alignments, an aligned pipeline composes them with + * cursor-based normalizers report alignments, an aligned pipeline composes them with * {@link Alignment#andThen(Alignment)} so a span found in the fully normalized text maps back to the - * original input, and a non-alignable rung is rejected loudly. + * original input, and a non-alignable normalizer is rejected loudly. */ public class AlignedNormalizerPipelineTest { @@ -50,8 +50,8 @@ private static String covered(AlignedText aligned, int normalizedStart, int norm // The aligned form must always reproduce exactly what the plain form produces. @Test - void alignedNormalizedTextMatchesPlainForEveryRung() { - final OffsetAwareNormalizer[] rungs = { + void alignedNormalizedTextMatchesPlainForEveryNormalizer() { + final OffsetAwareNormalizer[] normalizers = { WhitespaceCharSequenceNormalizer.getInstance(), LineBreakPreservingWhitespaceCharSequenceNormalizer.getInstance(), DashCharSequenceNormalizer.getInstance(), @@ -74,10 +74,10 @@ void alignedNormalizedTextMatchesPlainForEveryRung() { cp(0x201C) + "don" + cp(0x2019) + "t " + cp(0x2026) + " Stra" + cp(0x00DF) + "e " + cp(0x2022) + " " + cp(0xFF15) + cp(MATH_BOLD_DIGIT_ZERO) }; - for (final OffsetAwareNormalizer rung : rungs) { + for (final OffsetAwareNormalizer normalizer : normalizers) { for (final String input : inputs) { - assertEquals(rung.normalize(input).toString(), rung.normalizeAligned(input).normalized(), - rung.getClass().getSimpleName() + " on [" + input + "]"); + assertEquals(normalizer.normalize(input).toString(), normalizer.normalizeAligned(input).normalized(), + normalizer.getClass().getSimpleName() + " on [" + input + "]"); } } } @@ -137,7 +137,7 @@ void emptyAlignedPipelineIsIdentity() { } @Test - void buildAlignedRejectsNonAlignableRungLoudly() { + void buildAlignedRejectsNonAlignableNormalizerLoudly() { final IllegalStateException ex = assertThrows(IllegalStateException.class, () -> TextNormalizer.builder().nfc().whitespace().buildAligned()); assertTrue(ex.getMessage().contains("Nfc"), ex.getMessage()); @@ -145,7 +145,7 @@ void buildAlignedRejectsNonAlignableRungLoudly() { } @Test - void buildAlignedReportsTheOffendingRungIndexWhenItIsNotFirst() { + void buildAlignedReportsTheOffendingNormalizerIndexWhenItIsNotFirst() { // A non-alignable step after several offset-aware ones must still be rejected, and the message // must name its 0-based position (index 2) and type so the failure points at the right fold. final IllegalStateException ex = assertThrows(IllegalStateException.class, @@ -155,7 +155,7 @@ void buildAlignedReportsTheOffendingRungIndexWhenItIsNotFirst() { } @Test - void buildAlignedRejectsEachKindOfNonAlignableRung() { + void buildAlignedRejectsEachKindOfNonAlignableNormalizer() { // Every fold that routes through java.text.Normalizer or JDK case mapping is rejected, named. assertThrows(IllegalStateException.class, () -> TextNormalizer.builder().nfkc().buildAligned()); @@ -203,17 +203,17 @@ void roundTripOfAFullySpanningMatchReturnsTheWholeOriginal() { @Test void lineBreakPreservingCollapsesHorizontalRunsButKeepsBreaks() { - final LineBreakPreservingWhitespaceCharSequenceNormalizer rung = + final LineBreakPreservingWhitespaceCharSequenceNormalizer normalizer = LineBreakPreservingWhitespaceCharSequenceNormalizer.getInstance(); final String original = "Hello world\n\n\tfoo bar"; - assertEquals("Hello world\nfoo bar", rung.normalize(original).toString()); + assertEquals("Hello world\nfoo bar", normalizer.normalize(original).toString()); - // The plain whitespace rung instead flattens the blank line into a single space. + // The plain whitespace normalizer instead flattens the blank line into a single space. assertEquals("Hello world foo bar", WhitespaceCharSequenceNormalizer.getInstance().normalize(original).toString()); - final AlignedText aligned = rung.normalizeAligned(original); - assertEquals(rung.normalize(original).toString(), aligned.normalized()); + final AlignedText aligned = normalizer.normalizeAligned(original); + assertEquals(normalizer.normalize(original).toString(), aligned.normalized()); // "bar" sits at [16, 19) in the collapsed form and at [21, 24) in the original. assertEquals(original.indexOf("bar"), aligned.toOriginalSpan(16, 19).getStart()); assertEquals("bar", covered(aligned, 16, 19)); @@ -223,10 +223,10 @@ void lineBreakPreservingCollapsesHorizontalRunsButKeepsBreaks() { @Test void lineBreakPreservingTrimsLeadingAndTrailingBreaks() { - final LineBreakPreservingWhitespaceCharSequenceNormalizer rung = + final LineBreakPreservingWhitespaceCharSequenceNormalizer normalizer = LineBreakPreservingWhitespaceCharSequenceNormalizer.getInstance(); final String original = "\n\nHello\n\n"; - final AlignedText aligned = rung.normalizeAligned(original); + final AlignedText aligned = normalizer.normalizeAligned(original); assertEquals("Hello", aligned.normalized()); assertEquals("Hello", covered(aligned, 0, 5)); assertEquals(original.indexOf("Hello"), aligned.toOriginalSpan(0, 5).getStart()); @@ -261,18 +261,18 @@ void pipelineMapsAnOriginalSpanForwardToTheNormalizedText() { @Test void lineBreakPreservingNormalizesCrLfAndUnicodeSeparators() { - final LineBreakPreservingWhitespaceCharSequenceNormalizer rung = + final LineBreakPreservingWhitespaceCharSequenceNormalizer normalizer = LineBreakPreservingWhitespaceCharSequenceNormalizer.getInstance(); - assertEquals("a\nb", rung.normalize("a\r\nb").toString()); // CRLF -> one newline - assertEquals("a\nb", rung.normalize("a\n\n\n\nb").toString()); // blank lines -> one newline - assertEquals("x\ny", rung.normalize("x" + cp(0x2028) + "y").toString()); // line separator - assertEquals("p\nq", rung.normalize("p" + cp(0x2029) + "q").toString()); // paragraph separator + assertEquals("a\nb", normalizer.normalize("a\r\nb").toString()); // CRLF -> one newline + assertEquals("a\nb", normalizer.normalize("a\n\n\n\nb").toString()); // blank lines -> one newline + assertEquals("x\ny", normalizer.normalize("x" + cp(0x2028) + "y").toString()); // line separator + assertEquals("p\nq", normalizer.normalize("p" + cp(0x2029) + "q").toString()); // paragraph separator // A horizontal run still collapses to a space even when mixed with a break-bearing run. - assertEquals("a b\nc", rung.normalize("a b \n c").toString()); + assertEquals("a b\nc", normalizer.normalize("a b \n c").toString()); } @Test - void whitespaceRungCollapsesAllWhitespaceToEmptyWithAValidSpan() { + void whitespaceNormalizerCollapsesAllWhitespaceToEmptyWithAValidSpan() { final AlignedText aligned = WhitespaceCharSequenceNormalizer.getInstance().normalizeAligned(" "); assertEquals("", aligned.normalized()); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java index 6391a311a7..7845b12238 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TextNormalizerTest.java @@ -30,7 +30,7 @@ private static String cp(int codePoint) { } @Test - void testRungsApplyInOrder() { + void testNormalizersApplyInOrder() { final CharSequenceNormalizer n = TextNormalizer.builder().caseFold().accentFold().build(); assertEquals("cafe", n.normalize("CAF" + cp(0x00C9)).toString()); // CAFE-acute -> cafe } @@ -66,7 +66,7 @@ void testDefaultChainCleansMessyInput() { } @Test - void testEveryRungIsInvokable() { + void testEveryNormalizerIsInvokable() { final CharSequenceNormalizer n = TextNormalizer.builder() .stripInvisible().nfc().nfkc().whitespace().quotes().dashes().digits().ellipsis().bullets() .fullCaseFold().accentFold().build(); @@ -76,14 +76,14 @@ void testEveryRungIsInvokable() { } @Test - void testCaseFoldRungIsInvokable() { + void testCaseFoldNormalizerIsInvokable() { final CharSequenceNormalizer n = TextNormalizer.builder().caseFold().build(); assertEquals("cafe", n.normalize("CAFE").toString()); } @Test void testCaseFoldWithFullCaseFoldIsPermittedButRedundant() { - // This builder composes rungs freely and does not enforce the exclusion the TermAnalyzer + // This builder composes normalizers freely and does not enforce the exclusion the TermAnalyzer // layer does; the combination is documented as redundant, and this pins that it changes // nothing over full case folding alone. final String input = "STRA" + cp(0x00DF) + "E"; diff --git a/opennlp-docs/src/docbkx/normalizer.xml b/opennlp-docs/src/docbkx/normalizer.xml index d955e0366a..9a6d0e68f2 100644 --- a/opennlp-docs/src/docbkx/normalizer.xml +++ b/opennlp-docs/src/docbkx/normalizer.xml @@ -198,7 +198,7 @@ String hyphen = DashCharSequenceNormalizer.getInstance()TextNormalizer is a fluent builder that composes the rungs, in the order
+ TextNormalizer is a fluent builder that composes the normalizers, in the order
they are added, into a single CharSequenceNormalizer:
CharSequenceNormalizer and adds normalizeAligned, so a caller tests
for it with a plain instanceof, the same pattern the name finder uses for
OffsetMappingNameFinder. Every per-code-point fold implements it: whitespace, the
- line-break-preserving whitespace rung, dashes, invisible-control stripping, quotes, digits,
+ line-break-preserving whitespace normalizer, dashes, invisible-control stripping, quotes, digits,
ellipsis, bullets, the German umlaut transliteration, Unicode full case folding
(fullCaseFold(), whose expansions come from a bundled table with known lengths,
so it reports its edits), and the emoji/emoticon folds (emojiToEmoticon() and