From 6a71703c78d76ca32dff68649dec13c33e1b2330 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 10 Jul 2026 14:12:15 -0400 Subject: [PATCH 01/82] OPENNLP-1885: Add opennlp-subword: pure-Java SentencePiece inference with exact original-text spans New opennlp-extensions module implementing SentencePiece model inference without native code: the ModelProto reader, the model-embedded normalizer (precompiled character map over a Darts-clone double-array trie, whitespace collapsing and escaping, the dummy word-boundary marker), unigram best-path segmentation, BPE agenda merging, byte fallback, and user-defined symbol handling. The public contract is SubwordTokenizer/SubwordPiece; every piece reports the exact UTF-16 span of the caller's original text it came from, and the model normalizer is also exposed as an OffsetAwareNormalizer producing AlignedText. Parity with the reference implementation is asserted, not assumed: five tiny bundled models (unigram, unigram with byte fallback, BPE, identity normalization, whitespace-as-suffix) carry fixtures generated by the sentencepiece Python package over 40 inputs each, checked piece for piece, id for id, span for span, plus each model's embedded self-test samples. An opt-in test (-Dopennlp.subword.eval.dir) runs the same assertions against real downloaded models; T5-small and ALBERT-base-v2 pass exactly, including mixed scripts, emoji ZWJ sequences, BOM, and CRLF inputs. --- opennlp-extensions/opennlp-subword/pom.xml | 58 ++ .../java/opennlp/subword/SubwordPiece.java | 59 ++ .../opennlp/subword/SubwordTokenizer.java | 77 +++ .../subword/sentencepiece/BpeEncoder.java | 216 ++++++++ .../subword/sentencepiece/ByteBuilder.java | 64 +++ .../sentencepiece/DoubleArrayTrie.java | 95 ++++ .../subword/sentencepiece/IntBuilder.java | 56 ++ .../sentencepiece/ModelProtoReader.java | 265 +++++++++ .../subword/sentencepiece/PieceTrie.java | 208 +++++++ .../subword/sentencepiece/Segment.java | 27 + .../SentencePieceNormalizer.java | 320 +++++++++++ .../sentencepiece/SentencePieceTokenizer.java | 509 ++++++++++++++++++ .../subword/sentencepiece/UnigramEncoder.java | 163 ++++++ .../subword/sentencepiece/Utf8Text.java | 109 ++++ .../SentencePieceAlignmentTest.java | 145 +++++ .../SentencePieceModelValidationTest.java | 183 +++++++ .../SentencePieceParityTest.java | 150 ++++++ .../SentencePieceRealModelEvalTest.java | 112 ++++ .../opennlp/subword/sentencepiece/corpus.txt | 66 +++ .../subword/sentencepiece/gen_fixtures.py | 139 +++++ .../sentencepiece/gen_real_fixtures.py | 75 +++ .../sentencepiece/tiny-bpe.fixtures.tsv | 40 ++ .../subword/sentencepiece/tiny-bpe.model | Bin 0 -> 245063 bytes .../tiny-unigram-bytefb.fixtures.tsv | 40 ++ .../sentencepiece/tiny-unigram-bytefb.model | Bin 0 -> 250431 bytes .../tiny-unigram-identity.fixtures.tsv | 40 ++ .../sentencepiece/tiny-unigram-identity.model | Bin 0 -> 5287 bytes .../tiny-unigram-suffix.fixtures.tsv | 40 ++ .../sentencepiece/tiny-unigram-suffix.model | Bin 0 -> 245428 bytes .../sentencepiece/tiny-unigram.fixtures.tsv | 40 ++ .../subword/sentencepiece/tiny-unigram.model | Bin 0 -> 245202 bytes opennlp-extensions/pom.xml | 1 + rat-excludes | 5 + 33 files changed, 3302 insertions(+) create mode 100644 opennlp-extensions/opennlp-subword/pom.xml create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Segment.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java create mode 100644 opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/corpus.txt create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_fixtures.py create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_real_fixtures.py create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.fixtures.tsv create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.model create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.fixtures.tsv create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.model create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.fixtures.tsv create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.model create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.fixtures.tsv create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.model create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.fixtures.tsv create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.model diff --git a/opennlp-extensions/opennlp-subword/pom.xml b/opennlp-extensions/opennlp-subword/pom.xml new file mode 100644 index 0000000000..b4d13c755e --- /dev/null +++ b/opennlp-extensions/opennlp-subword/pom.xml @@ -0,0 +1,58 @@ + + + + + + 4.0.0 + + org.apache.opennlp + opennlp-extensions + 3.0.0-SNAPSHOT + + + opennlp-subword + jar + Apache OpenNLP :: Ext :: Subword + + + + org.apache.opennlp + opennlp-api + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java new file mode 100644 index 0000000000..1791583101 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword; + +import opennlp.tools.util.Span; + +/** + * One subword unit produced by a {@link SubwordTokenizer}, carrying both the vocabulary view + * (the piece string and its id) and the exact place in the caller's text it came from. + * + *

The piece string is in the tokenizer's internal, normalized form (for example, a leading + * word-boundary marker instead of a space), so it is generally not a substring of the input. + * {@code start} and {@code end} are UTF-16 offsets into the original input text, so the surface + * that produced this piece is {@code text.subSequence(start, end)}. Pieces that carry no surface + * of their own (control symbols, or the fill bytes of a byte-fallback expansion) report an empty + * span, {@code start == end}.

+ * + * @param piece The piece in the vocabulary's normalized form; never null or empty. + * @param id The vocabulary id of the piece. + * @param start The inclusive start offset in the original text. + * @param end The exclusive end offset in the original text; not less than {@code start}. + */ +public record SubwordPiece(String piece, int id, int start, int end) { + + /** + * Instantiates a {@link SubwordPiece}. + * + * @throws IllegalArgumentException Thrown if {@code piece} is null or empty, or the span is + * negative or inverted. + */ + public SubwordPiece { + if (piece == null || piece.isEmpty()) { + throw new IllegalArgumentException("The piece must not be null or empty."); + } + if (start < 0 || end < start) { + throw new IllegalArgumentException( + "The span [" + start + ", " + end + ") must not be negative or inverted."); + } + } + + /** {@return the original-text span of this piece as a {@link Span}} */ + public Span span() { + return new Span(start, end); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java new file mode 100644 index 0000000000..ea31342bf2 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword; + +import java.util.List; + +/** + * Splits text into subword units against a fixed vocabulary, reporting for every unit its + * vocabulary id and the exact span of the original text it covers. + * + *

Subword tokenization is the input layer of modern sequence models: text is decomposed into + * pieces from a trained vocabulary so that any input, including words never seen in training, maps + * to a bounded id space. Unlike a linguistic {@code Tokenizer}, the segmentation is + * vocabulary-driven, and the pieces are in the model's normalized form rather than substrings of + * the input. The offsets carried by each {@link SubwordPiece} are what tie the two worlds + * together: they always refer to the caller's original text.

+ * + *

Implementations are expected to be safe for concurrent use by multiple threads; any + * implementation that is not must document it.

+ */ +public interface SubwordTokenizer { + + /** + * Encodes text into subword pieces. + * + * @param text The text to encode; must not be null. + * @return The pieces in text order; empty when the text contains nothing encodable. + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ + List encode(CharSequence text); + + /** + * Encodes text into vocabulary ids. + * + * @param text The text to encode; must not be null. + * @return The ids in text order; empty when the text contains nothing encodable. + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ + default int[] encodeToIds(CharSequence text) { + final List pieces = encode(text); + final int[] ids = new int[pieces.size()]; + for (int i = 0; i < ids.length; i++) { + ids[i] = pieces.get(i).id(); + } + return ids; + } + + /** + * Encodes text into piece strings in the vocabulary's normalized form. + * + * @param text The text to encode; must not be null. + * @return The pieces in text order; empty when the text contains nothing encodable. + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ + default String[] encodeToPieces(CharSequence text) { + final List pieces = encode(text); + final String[] out = new String[pieces.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = pieces.get(i).piece(); + } + return out; + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java new file mode 100644 index 0000000000..3cea8edb64 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; + +/** + * Byte-pair-encoding segmentation: the normalized text starts as single characters (or + * user-defined symbols, which are frozen whole) and adjacent pairs merge greedily, highest piece + * score first, until no adjacent pair forms a vocabulary piece. + * + *

This ports the reference implementation's agenda algorithm: candidate pairs sit in a + * priority queue ordered by score with ties broken towards the leftmost pair, stale entries are + * detected by a length check when popped, and merges that land on a piece marked unused are + * re-segmented back into their constituents afterwards. Only pieces of the normal, user-defined, + * and unused types participate in merges.

+ */ +final class BpeEncoder { + + private static final int MAX_RESEGMENT_DEPTH = 100; + + private final Map pieces; + private final float[] scores; + private final boolean[] unused; + private final boolean[] reserved; + private final int unkId; + private final PieceTrie userDefinedMatcher; + + /** + * Instantiates the encoder. + * + * @param pieces All pieces by content, mapping to their ids. + * @param scores The score of every piece, indexed by id. + * @param unused Whether each id has the unused piece type. + * @param reserved Whether each id is excluded from merging (any type other than + * normal, user-defined, or unused). + * @param unkId The id of the unknown piece. + * @param userDefinedMatcher Longest-match trie over user-defined symbols, or null when the + * model defines none. + */ + BpeEncoder(Map pieces, float[] scores, boolean[] unused, boolean[] reserved, + int unkId, PieceTrie userDefinedMatcher) { + this.pieces = pieces; + this.scores = scores; + this.unused = unused; + this.reserved = reserved; + this.unkId = unkId; + this.userDefinedMatcher = userDefinedMatcher; + } + + // A candidate merge of the symbols at indices left and right; size is the merged byte length + // used to detect staleness after either side has changed. + private record Pair(int left, int right, float score, int size) { + } + + /** + * Segments normalized text. + * + * @param normalized The normalized UTF-8 bytes; must not be null. + * @return The segments covering all bytes, in text order. + */ + List encode(byte[] normalized) { + if (normalized.length == 0) { + return List.of(); + } + + // The symbol list as index-linked ranges of the normalized bytes; merged-away symbols + // become empty ranges. + final IntBuilder fromB = new IntBuilder(normalized.length); + final IntBuilder toB = new IntBuilder(normalized.length); + final List freezeList = new ArrayList<>(); + int position = 0; + while (position < normalized.length) { + int matched = 0; + if (userDefinedMatcher != null) { + matched = longestUserDefinedMatch(normalized, position); + } + final boolean frozen = matched > 0; + final int length = frozen ? matched + : Math.min(SentencePieceNormalizer.utf8Length(normalized[position]), + normalized.length - position); + fromB.append(position); + toB.append(position + length); + freezeList.add(frozen); + position += length; + } + final int symbolCount = freezeList.size(); + final int[] from = fromB.toArray(); + final int[] to = toB.toArray(); + final int[] prev = new int[symbolCount]; + final int[] next = new int[symbolCount]; + final boolean[] freeze = new boolean[symbolCount]; + for (int i = 0; i < symbolCount; i++) { + prev[i] = i - 1; + next[i] = i + 1 < symbolCount ? i + 1 : -1; + freeze[i] = freezeList.get(i); + } + + // Higher score first; equal scores break towards the leftmost pair. + final PriorityQueue agenda = new PriorityQueue<>((a, b) -> { + final int byScore = Float.compare(b.score(), a.score()); + return byScore != 0 ? byScore : Integer.compare(a.left(), b.left()); + }); + // Merged piece content mapped back to its two constituents, for re-segmenting unused pieces. + final Map revMerge = new HashMap<>(); + + for (int left = 0; left + 1 < symbolCount; left++) { + maybeAddPair(normalized, from, to, freeze, left, left + 1, agenda, revMerge); + } + + while (!agenda.isEmpty()) { + final Pair top = agenda.poll(); + // Skips entries made stale by an earlier merge of either side. + if (from[top.left()] == to[top.left()] || from[top.right()] == to[top.right()] + || to[top.left()] - from[top.left()] + to[top.right()] - from[top.right()] + != top.size()) { + continue; + } + + // Replaces the pair with the merged symbol. + to[top.left()] = to[top.right()]; + next[top.left()] = next[top.right()]; + if (next[top.right()] >= 0) { + prev[next[top.right()]] = top.left(); + } + from[top.right()] = to[top.right()]; + + maybeAddPair(normalized, from, to, freeze, prev[top.left()], top.left(), agenda, revMerge); + maybeAddPair(normalized, from, to, freeze, top.left(), next[top.left()], agenda, revMerge); + } + + final List output = new ArrayList<>(symbolCount); + int consumed = 0; + for (int index = 0; index != -1; index = next[index]) { + final String piece = + new String(normalized, from[index], to[index] - from[index], StandardCharsets.UTF_8); + consumed = resegment(piece, consumed, 0, revMerge, output); + } + return output; + } + + private void maybeAddPair(byte[] normalized, int[] from, int[] to, boolean[] freeze, + int left, int right, PriorityQueue agenda, + Map revMerge) { + if (left == -1 || right == -1 || freeze[left] || freeze[right]) { + return; + } + final String piece = + new String(normalized, from[left], to[right] - from[left], StandardCharsets.UTF_8); + final Integer id = pieces.get(piece); + if (id == null || id == unkId || reserved[id]) { + return; + } + agenda.add(new Pair(left, right, scores[id], to[right] - from[left])); + if (unused[id]) { + revMerge.put(piece, new String[] { + new String(normalized, from[left], to[left] - from[left], StandardCharsets.UTF_8), + new String(normalized, from[right], to[right] - from[right], StandardCharsets.UTF_8)}); + } + } + + // Emits a symbol, splitting a piece of the unused type back into the pieces it was merged + // from. Positions are assigned by a running cursor; constituent byte lengths always sum to the + // merged length, so the cursor stays aligned with the normalized bytes. + private int resegment(String piece, int consumed, int depth, Map revMerge, + List output) { + final Integer mapped = pieces.get(piece); + final int id = mapped == null ? unkId : mapped; + final int byteLength = piece.getBytes(StandardCharsets.UTF_8).length; + if (depth > MAX_RESEGMENT_DEPTH || !unused[id]) { + output.add(new Segment(consumed, consumed + byteLength, id)); + return consumed + byteLength; + } + final String[] parts = revMerge.get(piece); + if (parts == null) { + output.add(new Segment(consumed, consumed + byteLength, id)); + return consumed + byteLength; + } + consumed = resegment(parts[0], consumed, depth + 1, revMerge, output); + return resegment(parts[1], consumed, depth + 1, revMerge, output); + } + + private int longestUserDefinedMatch(byte[] input, int from) { + int node = userDefinedMatcher.root(); + int longest = 0; + for (int i = from; i < input.length; i++) { + node = userDefinedMatcher.step(node, input[i]); + if (node == PieceTrie.DEAD) { + break; + } + if (userDefinedMatcher.value(node) >= 0) { + longest = i - from + 1; + } + } + return longest; + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java new file mode 100644 index 0000000000..038b9f8ce1 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.util.Arrays; + +/** A growable byte buffer supporting append, truncate, and suffix comparison. */ +final class ByteBuilder { + + private byte[] data; + private int length; + + ByteBuilder(int capacity) { + data = new byte[Math.max(capacity, 16)]; + } + + void append(byte b) { + if (length == data.length) { + data = Arrays.copyOf(data, data.length + (data.length >> 1)); + } + data[length++] = b; + } + + void append(byte[] source, int from, int count) { + while (length + count > data.length) { + data = Arrays.copyOf(data, data.length + (data.length >> 1)); + } + System.arraycopy(source, from, data, length, count); + length += count; + } + + int length() { + return length; + } + + void truncate(int newLength) { + length = newLength; + } + + boolean endsWith(byte[] suffix) { + if (length < suffix.length) { + return false; + } + return Arrays.equals(data, length - suffix.length, length, suffix, 0, suffix.length); + } + + byte[] toArray() { + return Arrays.copyOf(data, length); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java new file mode 100644 index 0000000000..d824b87527 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +/** + * Read-only lookup over a serialized Darts-clone double-array trie, the dictionary format + * embedded in a SentencePiece model's precompiled character map. + * + *

Each unit is one little-endian 32-bit word encoding a label, an offset to the unit's + * children, and a leaf flag; traversal XORs the offset with the next key byte. Only the longest + * prefix match is needed here, so this walks the byte key once and remembers the last accepting + * state. Out-of-range unit references, which a well-formed trie never produces, fail loudly + * rather than reading arbitrary memory.

+ */ +final class DoubleArrayTrie { + + private final int[] units; + + /** + * Wraps serialized trie units. + * + * @param data The bytes holding the units; must not be null. + * @param offset The offset of the first unit byte. + * @param length The number of bytes; must be a positive multiple of four. + */ + DoubleArrayTrie(byte[] data, int offset, int length) { + if (length <= 0 || (length & 3) != 0) { + throw new IllegalArgumentException( + "The trie length " + length + " is not a positive multiple of four bytes."); + } + units = new int[length >> 2]; + for (int i = 0; i < units.length; i++) { + final int base = offset + (i << 2); + units[i] = (data[base] & 0xFF) | (data[base + 1] & 0xFF) << 8 + | (data[base + 2] & 0xFF) << 16 | (data[base + 3] & 0xFF) << 24; + } + } + + /** + * Finds the longest key that is a prefix of {@code key[from, to)}. + * + * @param key The byte key to match against; must not be null. + * @param from The inclusive start of the query window. + * @param to The exclusive end of the query window. + * @return {@code (value << 32) | matchedLength} for the longest match, or {@code -1} when no + * key matches. Values are non-negative, so the result is negative only on no-match. + */ + long longestPrefixMatch(byte[] key, int from, int to) { + long result = -1; + int nodePos = 0; + int unit = unit(nodePos); + nodePos ^= offset(unit); + for (int i = from; i < to; i++) { + final int b = key[i] & 0xFF; + nodePos ^= b; + unit = unit(nodePos); + if ((unit & 0x800000FF) != b) { + return result; + } + nodePos ^= offset(unit); + if (((unit >>> 8) & 1) == 1) { + final int value = unit(nodePos) & 0x7FFFFFFF; + result = ((long) value << 32) | (i - from + 1); + } + } + return result; + } + + private int unit(int nodePos) { + if (nodePos < 0 || nodePos >= units.length) { + throw new IllegalArgumentException( + "The trie references unit " + nodePos + " outside its " + units.length + " units."); + } + return units[nodePos]; + } + + // The offset from a unit to its children, as encoded by Darts-clone. + private static int offset(int unit) { + return (unit >>> 10) << ((unit & (1 << 9)) >>> 6); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java new file mode 100644 index 0000000000..70a2db9ac4 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.util.Arrays; + +/** A growable int buffer supporting append, indexed read, and truncate. */ +final class IntBuilder { + + private int[] data; + private int length; + + IntBuilder(int capacity) { + data = new int[Math.max(capacity, 16)]; + } + + void append(int value) { + if (length == data.length) { + data = Arrays.copyOf(data, data.length + (data.length >> 1)); + } + data[length++] = value; + } + + int get(int index) { + if (index >= length) { + throw new IndexOutOfBoundsException("index " + index + " is outside [0, " + length + ")"); + } + return data[index]; + } + + int length() { + return length; + } + + void truncate(int newLength) { + length = newLength; + } + + int[] toArray() { + return Arrays.copyOf(data, length); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java new file mode 100644 index 0000000000..4e75f49c97 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * Reads the binary {@code ModelProto} serialization of a SentencePiece {@code .model} file. + * + *

The format is standard protocol-buffer wire encoding of one flat message + * ({@code sentencepiece_model.proto}, Apache License 2.0), so this reader walks the tag stream + * directly and keeps only the fields inference needs: the pieces with scores and types, the + * normalizer spec, the handful of trainer-spec fields that change runtime behavior, and the + * embedded self-test samples. Unknown fields are skipped, malformed input fails loudly.

+ */ +final class ModelProtoReader { + + // Wire types of the protocol-buffer encoding. + private static final int WIRE_VARINT = 0; + private static final int WIRE_FIXED64 = 1; + private static final int WIRE_LEN = 2; + private static final int WIRE_FIXED32 = 5; + + private final byte[] data; + private int pos; + + private ModelProtoReader(byte[] data) { + this.data = data; + } + + /** + * Parses a serialized {@code ModelProto}. + * + * @param data The raw bytes of a {@code .model} file; must not be null. + * @return The parsed model description. + * @throws IllegalArgumentException Thrown if the bytes are not a well-formed model. + */ + static RawModel read(byte[] data) { + if (data == null) { + throw new IllegalArgumentException("The model data must not be null."); + } + final ModelProtoReader reader = new ModelProtoReader(data); + final RawModel model = new RawModel(); + while (reader.pos < data.length) { + final long tag = reader.varint(); + final int field = (int) (tag >>> 3); + switch (field) { + case 1 -> reader.piece(model, reader.lenPayload(tag)); + case 2 -> reader.trainerSpec(model, reader.lenPayload(tag)); + case 3 -> reader.normalizerSpec(model, reader.lenPayload(tag)); + case 4 -> reader.selfTestData(model, reader.lenPayload(tag)); + default -> reader.skip(tag); + } + } + if (model.pieces.isEmpty()) { + throw new IllegalArgumentException("The model defines no pieces."); + } + return model; + } + + private void piece(RawModel model, int end) { + String piece = null; + float score = 0; + int type = RawModel.TYPE_NORMAL; + while (pos < end) { + final long tag = varint(); + switch ((int) (tag >>> 3)) { + case 1 -> piece = utf8(lenPayload(tag)); + case 2 -> score = fixed32Float(tag); + case 3 -> type = (int) varintOf(tag); + default -> skip(tag); + } + } + if (piece == null || piece.isEmpty()) { + throw new IllegalArgumentException( + "The model contains an empty piece at index " + model.pieces.size() + "."); + } + if (Float.isNaN(score) || Float.isInfinite(score)) { + throw new IllegalArgumentException("The score of piece '" + piece + "' is not finite."); + } + model.pieces.add(piece); + model.scores.add(score); + model.types.add(type); + } + + private void trainerSpec(RawModel model, int end) { + while (pos < end) { + final long tag = varint(); + switch ((int) (tag >>> 3)) { + case 3 -> model.modelType = (int) varintOf(tag); + case 24 -> model.treatWhitespaceAsSuffix = varintOf(tag) != 0; + case 35 -> model.byteFallback = varintOf(tag) != 0; + case 40 -> model.unkId = (int) varintOf(tag); + default -> skip(tag); + } + } + } + + private void normalizerSpec(RawModel model, int end) { + while (pos < end) { + final long tag = varint(); + switch ((int) (tag >>> 3)) { + case 2 -> model.precompiledCharsMap = bytes(lenPayload(tag)); + case 3 -> model.addDummyPrefix = varintOf(tag) != 0; + case 4 -> model.removeExtraWhitespaces = varintOf(tag) != 0; + case 5 -> model.escapeWhitespaces = varintOf(tag) != 0; + default -> skip(tag); + } + } + } + + private void selfTestData(RawModel model, int end) { + while (pos < end) { + final long tag = varint(); + if ((int) (tag >>> 3) == 1) { + final int sampleEnd = lenPayload(tag); + String input = null; + String expected = null; + while (pos < sampleEnd) { + final long sampleTag = varint(); + switch ((int) (sampleTag >>> 3)) { + case 1 -> input = utf8(lenPayload(sampleTag)); + case 2 -> expected = utf8(lenPayload(sampleTag)); + default -> skip(sampleTag); + } + } + if (input != null && expected != null) { + model.selfTestInputs.add(input); + model.selfTestExpected.add(expected); + } + } else { + skip(tag); + } + } + } + + // Returns the exclusive end offset of a length-delimited payload, verifying the wire type. + private int lenPayload(long tag) { + if ((tag & 7) != WIRE_LEN) { + throw malformed("field " + (tag >>> 3) + " is not length-delimited"); + } + final long length = varint(); + if (length < 0 || pos + length > data.length) { + throw malformed("length " + length + " exceeds the remaining input"); + } + return pos + (int) length; + } + + private long varintOf(long tag) { + if ((tag & 7) != WIRE_VARINT) { + throw malformed("field " + (tag >>> 3) + " is not a varint"); + } + return varint(); + } + + private float fixed32Float(long tag) { + if ((tag & 7) != WIRE_FIXED32) { + throw malformed("field " + (tag >>> 3) + " is not a 32-bit value"); + } + if (pos + 4 > data.length) { + throw malformed("truncated 32-bit value"); + } + final int bits = (data[pos] & 0xFF) | (data[pos + 1] & 0xFF) << 8 + | (data[pos + 2] & 0xFF) << 16 | (data[pos + 3] & 0xFF) << 24; + pos += 4; + return Float.intBitsToFloat(bits); + } + + private String utf8(int end) { + final String s = new String(data, pos, end - pos, StandardCharsets.UTF_8); + pos = end; + return s; + } + + private byte[] bytes(int end) { + final byte[] b = new byte[end - pos]; + System.arraycopy(data, pos, b, 0, b.length); + pos = end; + return b; + } + + private long varint() { + long value = 0; + for (int shift = 0; shift < 64; shift += 7) { + if (pos >= data.length) { + throw malformed("truncated varint"); + } + final byte b = data[pos++]; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return value; + } + } + throw malformed("varint exceeds 64 bits"); + } + + private void skip(long tag) { + switch ((int) (tag & 7)) { + case WIRE_VARINT -> varint(); + case WIRE_FIXED64 -> advance(8); + case WIRE_LEN -> pos = lenPayload(tag); + case WIRE_FIXED32 -> advance(4); + default -> throw malformed("unsupported wire type " + (tag & 7)); + } + } + + private void advance(int count) { + if (pos + count > data.length) { + throw malformed("truncated field"); + } + pos += count; + } + + private IllegalArgumentException malformed(String detail) { + return new IllegalArgumentException( + "The model data is malformed at byte " + pos + ": " + detail + "."); + } + + /** The fields of a {@code ModelProto} that inference needs, with the proto's defaults. */ + static final class RawModel { + + static final int TYPE_NORMAL = 1; + static final int TYPE_UNKNOWN = 2; + static final int TYPE_CONTROL = 3; + static final int TYPE_USER_DEFINED = 4; + static final int TYPE_UNUSED = 5; + static final int TYPE_BYTE = 6; + + static final int MODEL_TYPE_UNIGRAM = 1; + static final int MODEL_TYPE_BPE = 2; + + final List pieces = new ArrayList<>(); + final List scores = new ArrayList<>(); + final List types = new ArrayList<>(); + + int modelType = MODEL_TYPE_UNIGRAM; + boolean byteFallback = false; + boolean treatWhitespaceAsSuffix = false; + int unkId = 0; + + byte[] precompiledCharsMap = new byte[0]; + boolean addDummyPrefix = true; + boolean removeExtraWhitespaces = true; + boolean escapeWhitespaces = true; + + final List selfTestInputs = new ArrayList<>(); + final List selfTestExpected = new ArrayList<>(); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java new file mode 100644 index 0000000000..eeb028e00b --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.util.Arrays; +import java.util.Comparator; + +/** + * An immutable byte-level trie over vocabulary pieces, packed into flat arrays. + * + *

Encoding walks it one byte at a time ({@link #step(int, byte)}) while scanning the input, so + * every piece that starts at a given input position is enumerated in one forward pass; this is the + * lattice-population step of subword segmentation. Children of a node are stored as a sorted + * label slice and found by binary search.

+ */ +final class PieceTrie { + + /** The node id returned when no transition exists. */ + static final int DEAD = -1; + + // Per node: the slice [childStart[n], childStart[n + 1]) of labels/childNodes, and the piece id + // accepted at the node, or -1. + private final int[] childStart; + private final byte[] labels; + private final int[] childNodes; + private final int[] values; + + private PieceTrie(int[] childStart, byte[] labels, int[] childNodes, int[] values) { + this.childStart = childStart; + this.labels = labels; + this.childNodes = childNodes; + this.values = values; + } + + /** + * Builds a trie from pieces and their ids. + * + * @param pieces The UTF-8 bytes of each piece; must not be null or contain empty keys. + * @param ids The id stored for each piece, parallel to {@code pieces}. + * @return The packed trie. + */ + static PieceTrie build(byte[][] pieces, int[] ids) { + final Integer[] order = new Integer[pieces.length]; + for (int i = 0; i < order.length; i++) { + order[i] = i; + } + Arrays.sort(order, Comparator.comparing(i -> pieces[i], Arrays::compareUnsigned)); + + // First pass counts nodes and edges, second pass fills the packed arrays; both walk the + // sorted keys with the same recursion, so the shapes agree by construction. + final Builder builder = new Builder(pieces, ids, order); + builder.count(0, pieces.length, 0); + builder.allocate(); + builder.fill(0, pieces.length, 0); + return new PieceTrie(builder.childStart, builder.labels, builder.childNodes, builder.values); + } + + /** {@return the root node id} */ + int root() { + return 0; + } + + /** + * Follows the transition labeled {@code b}. + * + * @param node The current node id. + * @param b The next key byte. + * @return The child node id, or {@link #DEAD} when no such transition exists. + */ + int step(int node, byte b) { + final int from = childStart[node]; + final int to = childStart[node + 1]; + int low = from; + int high = to - 1; + while (low <= high) { + final int mid = (low + high) >>> 1; + final int c = Byte.compareUnsigned(labels[mid], b); + if (c < 0) { + low = mid + 1; + } else if (c > 0) { + high = mid - 1; + } else { + return childNodes[mid]; + } + } + return DEAD; + } + + /** + * Returns the piece id accepted at a node. + * + * @param node The node id. + * @return The id, or {@code -1} when the node accepts no piece. + */ + int value(int node) { + return values[node]; + } + + // Builds the packed form from keys sorted by unsigned byte order. Key ranges sharing a prefix + // are contiguous after the sort, so each recursion partitions its range by the byte at the + // current depth. + private static final class Builder { + + private final byte[][] pieces; + private final int[] ids; + private final Integer[] order; + + private int nodeCount; + private int edgeCount; + + private int[] childStart; + private byte[] labels; + private int[] childNodes; + private int[] values; + private int nextNode; + private int nextEdge; + + Builder(byte[][] pieces, int[] ids, Integer[] order) { + this.pieces = pieces; + this.ids = ids; + this.order = order; + } + + void count(int from, int to, int depth) { + nodeCount++; + int i = from; + if (i < to && pieces[order[i]].length == depth) { + i++; + } + while (i < to) { + final byte label = pieces[order[i]][depth]; + int j = i; + while (j < to && pieces[order[j]][depth] == label) { + j++; + } + edgeCount++; + count(i, j, depth + 1); + i = j; + } + } + + void allocate() { + childStart = new int[nodeCount + 1]; + labels = new byte[edgeCount]; + childNodes = new int[edgeCount]; + values = new int[nodeCount]; + } + + int fill(int from, int to, int depth) { + final int node = nextNode++; + values[node] = -1; + int i = from; + if (i < to && pieces[order[i]].length == depth) { + if (values[node] != -1 || (i + 1 < to && pieces[order[i + 1]].length == depth)) { + throw new IllegalArgumentException( + "The piece '" + new String(pieces[order[i]], java.nio.charset.StandardCharsets.UTF_8) + + "' is defined more than once."); + } + values[node] = ids[order[i]]; + i++; + } + // Reserve this node's edge slice before recursing so siblings stay contiguous. + final int sliceStart = nextEdge; + int sliceCount = 0; + int scan = i; + while (scan < to) { + final byte label = pieces[order[scan]][depth]; + int j = scan; + while (j < to && pieces[order[j]][depth] == label) { + j++; + } + sliceCount++; + scan = j; + } + nextEdge += sliceCount; + childStart[node] = sliceStart; + childStart[node + 1] = nextEdge; + + int edge = sliceStart; + while (i < to) { + final byte label = pieces[order[i]][depth]; + int j = i; + while (j < to && pieces[order[j]][depth] == label) { + j++; + } + labels[edge] = label; + childNodes[edge] = fill(i, j, depth + 1); + edge++; + i = j; + } + return node; + } + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Segment.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Segment.java new file mode 100644 index 0000000000..7f603c6ac5 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Segment.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +/** + * One encoded piece as a half-open byte range of the normalized text plus its vocabulary id. + * + * @param from The inclusive start offset in the normalized bytes. + * @param to The exclusive end offset in the normalized bytes. + * @param id The vocabulary id; the unknown id when no piece covers the range. + */ +record Segment(int from, int to, int id) { +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java new file mode 100644 index 0000000000..4d30b425c1 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +/** + * The model-embedded text normalizer of a SentencePiece model, operating in UTF-8 byte space. + * + *

Normalization applies the model's precompiled character map (leftmost-longest replacement + * rules over UTF-8 prefixes), collapses and trims whitespace, optionally prepends the + * word-boundary marker, and escapes spaces to U+2581. Alongside the normalized bytes it produces + * {@code normToOrig}, mapping every normalized byte to the offset of the original byte chunk it + * was derived from, with one trailing entry for the end position; that map is what lets every + * downstream piece report an exact span of the caller's text.

+ * + *

This mirrors the reference implementation's normalizer semantics rule for rule, since parity + * of both the normalized bytes and the offset map is what the tests assert.

+ */ +final class SentencePieceNormalizer { + + // U+2581 LOWER ONE EIGHTH BLOCK in UTF-8, the escaped form of a space. + static final byte[] SPACE_SYMBOL = {(byte) 0xE2, (byte) 0x96, (byte) 0x81}; + + // U+FFFD REPLACEMENT CHARACTER in UTF-8, emitted for a malformed byte. + private static final byte[] REPLACEMENT_CHAR = {(byte) 0xEF, (byte) 0xBF, (byte) 0xBD}; + + private final DoubleArrayTrie trie; + private final byte[] blob; + private final int replacementsFrom; + private final boolean addDummyPrefix; + private final boolean removeExtraWhitespaces; + private final boolean escapeWhitespaces; + private final boolean treatWhitespaceAsSuffix; + private final PieceTrie userDefinedMatcher; + + /** + * Instantiates the normalizer. + * + * @param precompiledCharsMap The serialized character map; empty when the model has none. + * @param addDummyPrefix Whether a word-boundary marker is prepended. + * @param removeExtraWhitespaces Whether leading, trailing, and repeated whitespace collapses. + * @param escapeWhitespaces Whether spaces become U+2581. + * @param treatWhitespaceAsSuffix Whether the dummy marker is appended instead of prepended. + * @param userDefinedMatcher Longest-match trie over user-defined symbols that must pass + * through normalization untouched, or null when the model + * defines none. + * @throws IllegalArgumentException Thrown if the character map is structurally invalid. + */ + SentencePieceNormalizer(byte[] precompiledCharsMap, boolean addDummyPrefix, + boolean removeExtraWhitespaces, boolean escapeWhitespaces, + boolean treatWhitespaceAsSuffix, PieceTrie userDefinedMatcher) { + if (precompiledCharsMap.length == 0) { + trie = null; + blob = null; + replacementsFrom = 0; + } else { + // Layout: . + if (precompiledCharsMap.length <= 4) { + throw new IllegalArgumentException("The precompiled character map is truncated."); + } + final long trieSize = (precompiledCharsMap[0] & 0xFFL) + | (precompiledCharsMap[1] & 0xFFL) << 8 + | (precompiledCharsMap[2] & 0xFFL) << 16 + | (precompiledCharsMap[3] & 0xFFL) << 24; + if (trieSize >= precompiledCharsMap.length - 4) { + throw new IllegalArgumentException( + "The precompiled character map declares a trie of " + trieSize + + " bytes but only " + (precompiledCharsMap.length - 4) + " bytes follow."); + } + if (trieSize < 1024 || (trieSize & 0x3FF) != 0) { + throw new IllegalArgumentException( + "The precompiled character map trie size " + trieSize + + " is not a positive multiple of 1024."); + } + if (precompiledCharsMap[precompiledCharsMap.length - 1] != 0) { + throw new IllegalArgumentException( + "The precompiled character map replacement block is not null-terminated."); + } + trie = new DoubleArrayTrie(precompiledCharsMap, 4, (int) trieSize); + blob = precompiledCharsMap; + replacementsFrom = 4 + (int) trieSize; + } + this.addDummyPrefix = addDummyPrefix; + this.removeExtraWhitespaces = removeExtraWhitespaces; + this.escapeWhitespaces = escapeWhitespaces; + this.treatWhitespaceAsSuffix = treatWhitespaceAsSuffix; + this.userDefinedMatcher = userDefinedMatcher; + } + + /** The normalized bytes plus the normalized-byte to original-byte offset map. */ + record Normalized(byte[] bytes, int[] normToOrig) { + } + + // One normalization step: `consumed` input bytes produced `data[from, to)`. The data array is + // the input itself (pass-through), the replacement blob, or the replacement character. + private record Chunk(byte[] data, int from, int to, int consumed) { + + boolean isSingleSpace() { + return to - from == 1 && data[from] == ' '; + } + } + + /** + * Normalizes UTF-8 input. + * + * @param input The well-formed UTF-8 bytes to normalize; must not be null. + * @return The normalized bytes with the offset map; {@code normToOrig.length} is always + * {@code bytes.length + 1}. + */ + Normalized normalize(byte[] input) { + final ByteBuilder normalized = new ByteBuilder(input.length + (input.length >> 1) + 4); + final IntBuilder normToOrig = new IntBuilder(input.length + (input.length >> 1) + 5); + + int from = 0; + int consumed = 0; + + // Ignores heading whitespace. + if (removeExtraWhitespaces) { + while (from < input.length) { + final Chunk p = normalizePrefix(input, from); + if (!p.isSingleSpace()) { + break; + } + from += p.consumed(); + consumed += p.consumed(); + } + } + + // All input was whitespace. + if (from >= input.length) { + return new Normalized(new byte[0], new int[] {consumed}); + } + + final byte[] spaceSymbol = escapeWhitespaces ? SPACE_SYMBOL : new byte[] {' '}; + + if (!treatWhitespaceAsSuffix && addDummyPrefix) { + appendSpace(normalized, normToOrig, spaceSymbol, consumed); + } + + boolean isPrevSpace = removeExtraWhitespaces; + while (from < input.length) { + final Chunk p = normalizePrefix(input, from); + int spFrom = p.from(); + final int spTo = p.to(); + final byte[] spData = p.data(); + + // Removes heading spaces in the chunk if the previous chunk ended with whitespace. + while (isPrevSpace && spFrom < spTo && spData[spFrom] == ' ') { + spFrom++; + } + + if (spFrom < spTo) { + for (int n = spFrom; n < spTo; n++) { + if (spData[n] == ' ') { + appendSpace(normalized, normToOrig, spaceSymbol, consumed); + } else { + normalized.append(spData[n]); + normToOrig.append(consumed); + } + } + isPrevSpace = spData[spTo - 1] == ' '; + } + + consumed += p.consumed(); + from += p.consumed(); + if (!removeExtraWhitespaces) { + isPrevSpace = false; + } + } + + // Ignores trailing whitespace. + if (removeExtraWhitespaces) { + while (normalized.endsWith(spaceSymbol)) { + final int length = normalized.length() - spaceSymbol.length; + consumed = normToOrig.get(length); + normalized.truncate(length); + normToOrig.truncate(length); + } + } + + if (treatWhitespaceAsSuffix && addDummyPrefix) { + appendSpace(normalized, normToOrig, spaceSymbol, consumed); + } + + normToOrig.append(consumed); + if (normToOrig.length() != normalized.length() + 1) { + throw new IllegalStateException("The offset map has " + normToOrig.length() + + " entries for " + normalized.length() + " normalized bytes."); + } + return new Normalized(normalized.toArray(), normToOrig.toArray()); + } + + private static void appendSpace(ByteBuilder normalized, IntBuilder normToOrig, + byte[] spaceSymbol, int consumed) { + normalized.append(spaceSymbol, 0, spaceSymbol.length); + for (int i = 0; i < spaceSymbol.length; i++) { + normToOrig.append(consumed); + } + } + + // Normalizes the longest applicable prefix of input[from, ...): a user-defined symbol passes + // through raw, otherwise the longest character-map rule applies, otherwise one code point + // passes through raw (or becomes U+FFFD when the lead byte is malformed). + private Chunk normalizePrefix(byte[] input, int from) { + if (userDefinedMatcher != null) { + final int matched = longestUserDefinedMatch(input, from); + if (matched > 0) { + return new Chunk(input, from, from + matched, matched); + } + } + + if (trie != null) { + final long match = trie.longestPrefixMatch(input, from, input.length); + if (match >= 0) { + final int value = (int) (match >>> 32); + final int length = (int) (match & 0xFFFFFFFFL); + final int replacementFrom = replacementsFrom + value; + if (replacementFrom < blob.length) { + int replacementTo = replacementFrom; + while (blob[replacementTo] != 0) { + replacementTo++; + } + return new Chunk(blob, replacementFrom, replacementTo, length); + } + } + } + + final int charLength = Math.min(utf8Length(input[from]), input.length - from); + if (isMalformed(input, from, charLength)) { + return new Chunk(REPLACEMENT_CHAR, 0, REPLACEMENT_CHAR.length, 1); + } + return new Chunk(input, from, from + charLength, charLength); + } + + private int longestUserDefinedMatch(byte[] input, int from) { + int node = userDefinedMatcher.root(); + int longest = 0; + for (int i = from; i < input.length; i++) { + node = userDefinedMatcher.step(node, input[i]); + if (node == PieceTrie.DEAD) { + break; + } + if (userDefinedMatcher.value(node) >= 0) { + longest = i - from + 1; + } + } + return longest; + } + + // The byte length of a UTF-8 sequence by its lead byte, as the reference implementation + // computes it: trail and malformed lead bytes report one byte. + static int utf8Length(byte lead) { + final int high = (lead & 0xFF) >>> 4; + if (high < 0xC) { + return 1; + } + return switch (high) { + case 0xC, 0xD -> 2; + case 0xE -> 3; + default -> 4; + }; + } + + // Checks a single code point for well-formedness: correct trail-byte count and no unpaired + // surrogate or out-of-range value. The public tokenizer API encodes its own well-formed UTF-8, + // so this only guards direct byte-level use. + private static boolean isMalformed(byte[] input, int from, int length) { + if ((input[from] & 0x80) == 0) { + return false; + } + if ((input[from] & 0xC0) == 0x80 || length < utf8Length(input[from])) { + return true; + } + for (int i = from + 1; i < from + length; i++) { + if ((input[i] & 0xC0) != 0x80) { + return true; + } + } + final int codePoint = codePointAt(input, from, length); + return codePoint < 0 || (codePoint >= 0xD800 && codePoint <= 0xDFFF) || codePoint > 0x10FFFF + || length != minimalUtf8Length(codePoint); + } + + private static int codePointAt(byte[] input, int from, int length) { + return switch (length) { + case 1 -> input[from] & 0x7F; + case 2 -> (input[from] & 0x1F) << 6 | (input[from + 1] & 0x3F); + case 3 -> (input[from] & 0x0F) << 12 | (input[from + 1] & 0x3F) << 6 + | (input[from + 2] & 0x3F); + default -> (input[from] & 0x07) << 18 | (input[from + 1] & 0x3F) << 12 + | (input[from + 2] & 0x3F) << 6 | (input[from + 3] & 0x3F); + }; + } + + private static int minimalUtf8Length(int codePoint) { + if (codePoint < 0x80) { + return 1; + } + if (codePoint < 0x800) { + return 2; + } + if (codePoint < 0x10000) { + return 3; + } + return 4; + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java new file mode 100644 index 0000000000..12e8271d76 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.subword.SubwordPiece; +import opennlp.subword.SubwordTokenizer; +import opennlp.tools.util.normalizer.AlignedText; +import opennlp.tools.util.normalizer.Alignment; +import opennlp.tools.util.normalizer.OffsetAwareNormalizer; + +/** + * A {@link SubwordTokenizer} over a trained SentencePiece model file, implemented purely in Java. + * + *

A {@code .model} file is self-contained: it carries the vocabulary with piece scores and + * types, the segmentation algorithm (unigram language model or byte-pair encoding), and the text + * normalizer the model was trained with. This class runs all three, so its output matches the + * reference implementation piece for piece and id for id, which is what makes the produced ids + * valid inputs for models trained against the same vocabulary.

+ * + *

Beyond parity, every piece carries the exact span of the caller's original text it came + * from, mapped back through the model's own normalizer. The normalizer is also exposed on its own + * through {@link OffsetAwareNormalizer}, so the model's text normalization can be reused as an + * offset-aware step outside of tokenization.

+ * + *

Instances are immutable after loading and safe for concurrent use by multiple threads.

+ */ +public final class SentencePieceTokenizer implements SubwordTokenizer, OffsetAwareNormalizer { + + /** The segmentation algorithm a model was trained with. */ + public enum Algorithm { + /** Unigram language model, decoded by best-path search. */ + UNIGRAM, + /** Byte-pair encoding, decoded by greedy highest-score merging. */ + BPE + } + + // Piece types of the model format. + private static final int TYPE_NORMAL = ModelProtoReader.RawModel.TYPE_NORMAL; + private static final int TYPE_UNKNOWN = ModelProtoReader.RawModel.TYPE_UNKNOWN; + private static final int TYPE_CONTROL = ModelProtoReader.RawModel.TYPE_CONTROL; + private static final int TYPE_USER_DEFINED = ModelProtoReader.RawModel.TYPE_USER_DEFINED; + private static final int TYPE_UNUSED = ModelProtoReader.RawModel.TYPE_UNUSED; + private static final int TYPE_BYTE = ModelProtoReader.RawModel.TYPE_BYTE; + + private static final int MAX_PIECE_LENGTH = 8000; + + private final Algorithm algorithm; + private final String[] pieces; + private final float[] scores; + private final int[] types; + private final int unkId; + private final boolean byteFallback; + private final Map mainPieces; + private final Map reservedPieces; + private final int[] byteToId; + private final SentencePieceNormalizer normalizer; + private final UnigramEncoder unigramEncoder; + private final BpeEncoder bpeEncoder; + private final List selfTestInputs; + private final List selfTestExpected; + + private SentencePieceTokenizer(ModelProtoReader.RawModel model) { + final int count = model.pieces.size(); + pieces = model.pieces.toArray(new String[0]); + scores = new float[count]; + types = new int[count]; + for (int i = 0; i < count; i++) { + scores[i] = model.scores.get(i); + types[i] = model.types.get(i); + } + byteFallback = model.byteFallback; + algorithm = switch (model.modelType) { + case ModelProtoReader.RawModel.MODEL_TYPE_UNIGRAM -> Algorithm.UNIGRAM; + case ModelProtoReader.RawModel.MODEL_TYPE_BPE -> Algorithm.BPE; + default -> throw new IllegalArgumentException( + "The model type " + model.modelType + " is not supported; only the unigram and BPE" + + " algorithms are."); + }; + + // Splits the vocabulary the way the reference does: pieces of the normal, user-defined, and + // unused types participate in segmentation, all others are reserved ids. + mainPieces = new HashMap<>(count * 2); + reservedPieces = new HashMap<>(); + final List userDefined = new ArrayList<>(); + byteToId = new int[256]; + java.util.Arrays.fill(byteToId, -1); + int foundUnkId = -1; + float minScore = Float.MAX_VALUE; + for (int i = 0; i < count; i++) { + final String piece = pieces[i]; + if (piece.length() >= MAX_PIECE_LENGTH) { + throw new IllegalArgumentException("The piece with id " + i + " is longer than " + + MAX_PIECE_LENGTH + " characters."); + } + if (piece.indexOf(0) >= 0) { + throw new IllegalArgumentException( + "The piece with id " + i + " contains a null character."); + } + final boolean isMain = + types[i] == TYPE_NORMAL || types[i] == TYPE_USER_DEFINED || types[i] == TYPE_UNUSED; + final Map target = + isMain || algorithm == Algorithm.BPE ? mainPieces : reservedPieces; + if (mainPieces.containsKey(piece) || reservedPieces.containsKey(piece)) { + throw new IllegalArgumentException("The piece '" + piece + "' is defined more than once."); + } + target.put(piece, i); + switch (types[i]) { + case TYPE_NORMAL -> minScore = Math.min(minScore, scores[i]); + case TYPE_USER_DEFINED -> userDefined.add(piece); + case TYPE_UNKNOWN -> { + if (foundUnkId >= 0) { + throw new IllegalArgumentException("The model defines more than one unknown piece."); + } + foundUnkId = i; + } + case TYPE_BYTE -> { + if (!byteFallback) { + throw new IllegalArgumentException("The model defines the byte piece '" + piece + + "' although byte fallback is disabled."); + } + final int b = parseBytePiece(piece); + if (b < 0) { + throw new IllegalArgumentException("The byte piece '" + piece + "' is invalid."); + } + byteToId[b] = i; + } + default -> { + // CONTROL and UNUSED need no bookkeeping here. + } + } + } + if (foundUnkId < 0) { + throw new IllegalArgumentException("The model defines no unknown piece."); + } + unkId = foundUnkId; + if (byteFallback) { + for (int b = 0; b < 256; b++) { + if (byteToId[b] < 0) { + throw new IllegalArgumentException("The model enables byte fallback but defines no" + + " piece for byte " + b + "."); + } + } + } + + final PieceTrie userDefinedMatcher = userDefined.isEmpty() ? null : trieOf(userDefined, id -> 0); + + normalizer = new SentencePieceNormalizer(model.precompiledCharsMap, model.addDummyPrefix, + model.removeExtraWhitespaces, model.escapeWhitespaces, model.treatWhitespaceAsSuffix, + userDefinedMatcher); + + final boolean[] unusedFlags = new boolean[count]; + final boolean[] userDefinedFlags = new boolean[count]; + final boolean[] reservedFlags = new boolean[count]; + for (int i = 0; i < count; i++) { + unusedFlags[i] = types[i] == TYPE_UNUSED; + userDefinedFlags[i] = types[i] == TYPE_USER_DEFINED; + reservedFlags[i] = types[i] != TYPE_NORMAL && types[i] != TYPE_USER_DEFINED + && types[i] != TYPE_UNUSED; + } + + if (algorithm == Algorithm.UNIGRAM) { + final List mainList = new ArrayList<>(mainPieces.size()); + final List mainIds = new ArrayList<>(mainPieces.size()); + for (int i = 0; i < count; i++) { + if (!reservedFlags[i]) { + mainList.add(pieces[i]); + mainIds.add(i); + } + } + final PieceTrie vocabulary = trieOf(mainList, mainIds::get); + unigramEncoder = new UnigramEncoder(vocabulary, scores, unusedFlags, userDefinedFlags, + minScore, unkId); + bpeEncoder = null; + } else { + unigramEncoder = null; + bpeEncoder = new BpeEncoder(mainPieces, scores, unusedFlags, reservedFlags, unkId, + userDefinedMatcher); + } + + selfTestInputs = List.copyOf(model.selfTestInputs); + selfTestExpected = List.copyOf(model.selfTestExpected); + } + + private static PieceTrie trieOf(List pieceList, + java.util.function.IntUnaryOperator idOf) { + final byte[][] keys = new byte[pieceList.size()][]; + final int[] ids = new int[pieceList.size()]; + for (int i = 0; i < keys.length; i++) { + keys[i] = pieceList.get(i).getBytes(StandardCharsets.UTF_8); + ids[i] = idOf.applyAsInt(i); + } + return PieceTrie.build(keys, ids); + } + + /** + * Loads a model from a file. + * + * @param modelFile The {@code .model} file to load; must not be null. + * @return The ready-to-use tokenizer. + * @throws IOException Thrown if the file cannot be read. + * @throws IllegalArgumentException Thrown if the file is not a valid model. + */ + public static SentencePieceTokenizer load(Path modelFile) throws IOException { + if (modelFile == null) { + throw new IllegalArgumentException("The model file must not be null."); + } + return new SentencePieceTokenizer(ModelProtoReader.read(Files.readAllBytes(modelFile))); + } + + /** + * Loads a model from a stream. The stream is read fully but not closed. + * + * @param in The stream positioned at the start of a {@code .model} serialization; must not be + * null. + * @return The ready-to-use tokenizer. + * @throws IOException Thrown if the stream cannot be read. + * @throws IllegalArgumentException Thrown if the bytes are not a valid model. + */ + public static SentencePieceTokenizer load(InputStream in) throws IOException { + if (in == null) { + throw new IllegalArgumentException("The input stream must not be null."); + } + return new SentencePieceTokenizer(ModelProtoReader.read(in.readAllBytes())); + } + + @Override + public List encode(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + final Utf8Text input = Utf8Text.of(text); + final SentencePieceNormalizer.Normalized normalized = normalizer.normalize(input.bytes()); + final List segments = algorithm == Algorithm.UNIGRAM + ? unigramEncoder.encode(normalized.bytes()) + : bpeEncoder.encode(normalized.bytes()); + + final List out = new ArrayList<>(segments.size()); + final byte[] norm = normalized.bytes(); + final int[] normToOrig = normalized.normToOrig(); + + // Accumulates a run of adjacent unknown pieces into one, as the reference does, so a decoder + // sees a single unknown token per unknown region. + StringBuilder pendingUnk = null; + int pendingUnkStart = 0; + int pendingUnkEnd = 0; + + for (final Segment segment : segments) { + final String piece = new String(norm, segment.from(), segment.to() - segment.from(), + StandardCharsets.UTF_8); + final boolean isUnk = segment.id() == unkId; + final boolean isControl = types[segment.id()] == TYPE_CONTROL; + + if (isControl) { + if (pendingUnk != null) { + out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd)); + pendingUnk = null; + } + final int at = input.charOffset(normToOrig[segment.from()]); + out.add(new SubwordPiece(piece, segment.id(), at, at)); + continue; + } + + final int origBegin = input.charOffset(normToOrig[segment.from()]); + final int origEnd = input.charOffset(normToOrig[segment.to()]); + + if (isUnk && byteFallback) { + if (pendingUnk != null) { + out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd)); + pendingUnk = null; + } + // Decomposes the unknown region into byte pieces; the last one carries the surface span. + for (int i = segment.from(); i < segment.to(); i++) { + final int b = norm[i] & 0xFF; + final boolean last = i == segment.to() - 1; + out.add(new SubwordPiece(BYTE_PIECES[b], byteToId[b], origBegin, + last ? origEnd : origBegin)); + } + } else if (isUnk) { + if (pendingUnk == null) { + pendingUnk = new StringBuilder(piece); + pendingUnkStart = origBegin; + } else { + pendingUnk.append(piece); + } + pendingUnkEnd = origEnd; + } else { + if (pendingUnk != null) { + out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd)); + pendingUnk = null; + } + out.add(new SubwordPiece(piece, segment.id(), origBegin, origEnd)); + } + } + if (pendingUnk != null) { + out.add(new SubwordPiece(pendingUnk.toString(), unkId, pendingUnkStart, pendingUnkEnd)); + } + return out; + } + + @Override + public CharSequence normalize(CharSequence text) { + return normalizeAligned(text).normalized(); + } + + @Override + public AlignedText normalizeAligned(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + final Utf8Text input = Utf8Text.of(text); + final SentencePieceNormalizer.Normalized result = normalizer.normalize(input.bytes()); + final String normalized = new String(result.bytes(), StandardCharsets.UTF_8); + final int[] normToOrig = result.normToOrig(); + final byte[] norm = result.bytes(); + + // Walks the normalized code points, grouping neighbors that came from the same original + // block into one replace run; gaps between blocks are deletions. + final Alignment.Builder builder = new Alignment.Builder(); + int cursor = 0; + int groupOrigStart = -1; + int groupOrigEnd = -1; + int groupChars = 0; + int b = 0; + while (b < norm.length) { + final int byteLength = Math.min(SentencePieceNormalizer.utf8Length(norm[b]), + norm.length - b); + final int origStart = input.charOffset(normToOrig[b]); + final int origEnd = input.charOffset(normToOrig[b + byteLength]); + final int chars = byteLength == 4 ? 2 : 1; + if (groupChars > 0 && origStart == groupOrigStart && origEnd == groupOrigEnd) { + groupChars += chars; + } else { + cursor = flushGroup(builder, cursor, groupOrigStart, groupOrigEnd, groupChars); + groupOrigStart = origStart; + groupOrigEnd = origEnd; + groupChars = chars; + } + b += byteLength; + } + cursor = flushGroup(builder, cursor, groupOrigStart, groupOrigEnd, groupChars); + if (cursor < input.charLength()) { + builder.replace(input.charLength() - cursor, 0); + } + return new AlignedText(text, normalized, builder.build(input.charLength())); + } + + private static int flushGroup(Alignment.Builder builder, int cursor, int origStart, int origEnd, + int chars) { + if (chars == 0) { + return cursor; + } + if (origStart > cursor) { + builder.replace(origStart - cursor, 0); + } + builder.replace(origEnd - Math.max(origStart, cursor), chars); + return Math.max(origEnd, cursor); + } + + /** {@return the segmentation algorithm of the loaded model} */ + public Algorithm algorithm() { + return algorithm; + } + + /** {@return the number of pieces in the vocabulary} */ + public int vocabularySize() { + return pieces.length; + } + + /** + * Returns the piece string of an id. + * + * @param id A vocabulary id in {@code [0, vocabularySize())}. + * @return The piece string. + * @throws IllegalArgumentException Thrown if {@code id} is out of range. + */ + public String idToPiece(int id) { + checkId(id); + return pieces[id]; + } + + /** + * Returns the id of a piece string. + * + * @param piece The piece to look up; must not be null. + * @return The id, or the unknown id when the vocabulary does not contain the piece. + */ + public int pieceToId(String piece) { + if (piece == null) { + throw new IllegalArgumentException("The piece must not be null."); + } + final Integer reserved = reservedPieces.get(piece); + if (reserved != null) { + return reserved; + } + return mainPieces.getOrDefault(piece, unkId); + } + + /** + * Returns the score of a piece. + * + * @param id A vocabulary id in {@code [0, vocabularySize())}. + * @return The score; a log-probability for unigram models, a merge rank for BPE models. + * @throws IllegalArgumentException Thrown if {@code id} is out of range. + */ + public float score(int id) { + checkId(id); + return scores[id]; + } + + /** {@return the id of the unknown piece} */ + public int unknownId() { + return unkId; + } + + /** + * Checks whether an id is the unknown piece. + * + * @param id A vocabulary id in {@code [0, vocabularySize())}. + * @return {@code true} for the unknown piece. + * @throws IllegalArgumentException Thrown if {@code id} is out of range. + */ + public boolean isUnknown(int id) { + checkId(id); + return types[id] == TYPE_UNKNOWN; + } + + /** + * Checks whether an id is a control piece. + * + * @param id A vocabulary id in {@code [0, vocabularySize())}. + * @return {@code true} for control pieces. + * @throws IllegalArgumentException Thrown if {@code id} is out of range. + */ + public boolean isControl(int id) { + checkId(id); + return types[id] == TYPE_CONTROL; + } + + /** + * Checks whether an id is a byte-fallback piece. + * + * @param id A vocabulary id in {@code [0, vocabularySize())}. + * @return {@code true} for byte pieces. + * @throws IllegalArgumentException Thrown if {@code id} is out of range. + */ + public boolean isByte(int id) { + checkId(id); + return types[id] == TYPE_BYTE; + } + + private void checkId(int id) { + if (id < 0 || id >= pieces.length) { + throw new IllegalArgumentException( + "The id " + id + " is outside [0, " + pieces.length + ")."); + } + } + + // The embedded self-test samples, exposed for the parity tests. + List selfTestInputs() { + return selfTestInputs; + } + + List selfTestExpected() { + return selfTestExpected; + } + + // "<0xAB>" piece strings for all byte values, as byte fallback emits them. + private static final String[] BYTE_PIECES = new String[256]; + + static { + final char[] hex = "0123456789ABCDEF".toCharArray(); + for (int b = 0; b < 256; b++) { + BYTE_PIECES[b] = "<0x" + hex[b >>> 4] + hex[b & 0xF] + ">"; + } + } + + private static int parseBytePiece(String piece) { + if (piece.length() != 6 || !piece.startsWith("<0x") || piece.charAt(5) != '>') { + return -1; + } + final int high = Character.digit(piece.charAt(3), 16); + final int low = Character.digit(piece.charAt(4), 16); + return high < 0 || low < 0 ? -1 : (high << 4) | low; + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java new file mode 100644 index 0000000000..a53ca70740 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Viterbi segmentation under a unigram language model: of all ways to cover the normalized text + * with vocabulary pieces, it finds the one with the highest total log-probability. + * + *

This is a port of the reference implementation's optimized single-pass decoder, which + * exploits the unigram independence assumption to keep only the best path ending at each byte + * position instead of a full lattice. Characters no piece covers fall back to the unknown id + * with a fixed penalty below the lowest piece score, and user-defined symbols receive a + * length-based bonus score so they always win. Tie-breaking and score arithmetic follow the + * reference exactly, including its occasional re-basing of accumulated scores on very long + * inputs, because segmentation parity is asserted against it.

+ */ +final class UnigramEncoder { + + private static final float UNK_PENALTY = 10.0f; + private static final float SCORE_RESET_THRESHOLD = 100000.0f; + + private final PieceTrie trie; + private final float[] scores; + private final boolean[] unused; + private final boolean[] userDefined; + private final float unkScore; + private final int unkId; + + /** + * Instantiates the encoder. + * + * @param trie The vocabulary trie over all matchable pieces. + * @param scores The log-probability score of every piece, indexed by id. + * @param unused Whether each id has the unused piece type. + * @param userDefined Whether each id is a user-defined symbol. + * @param minScore The lowest score among normal pieces. + * @param unkId The id of the unknown piece. + */ + UnigramEncoder(PieceTrie trie, float[] scores, boolean[] unused, boolean[] userDefined, + float minScore, int unkId) { + this.trie = trie; + this.scores = scores; + this.unused = unused; + this.userDefined = userDefined; + this.unkScore = minScore - UNK_PENALTY; + this.unkId = unkId; + } + + /** + * Segments normalized text. + * + * @param normalized The normalized UTF-8 bytes; must not be null. + * @return The best-path segments covering all bytes, in text order. + */ + List encode(byte[] normalized) { + final int size = normalized.length; + if (size == 0) { + return List.of(); + } + + // The best path ending at each byte position (exclusive end). + final int[] bestStartsAt = new int[size + 1]; + final float[] bestScore = new float[size + 1]; + final int[] bestId = new int[size + 1]; + java.util.Arrays.fill(bestStartsAt, -1); + + int startsAt = 0; + int maxFrontier = 0; + while (startsAt < size) { + float bestScoreTillHere = bestScore[startsAt]; + if (bestScoreTillHere < -SCORE_RESET_THRESHOLD + || bestScoreTillHere > SCORE_RESET_THRESHOLD) { + // Re-bases accumulated scores to keep float precision on very long inputs; every + // reachable frontier position shifts by the same offset, so the argmax is unchanged. + final float offset = bestScoreTillHere; + for (int i = startsAt; i <= maxFrontier; i++) { + if (i == startsAt || bestStartsAt[i] != -1) { + bestScore[i] -= offset; + } + } + bestScoreTillHere = 0.0f; + } + + boolean hasSingleNode = false; + final int mblen = Math.min(SentencePieceNormalizer.utf8Length(normalized[startsAt]), + size - startsAt); + + int node = trie.root(); + for (int keyPos = startsAt; keyPos < size; ) { + node = trie.step(node, normalized[keyPos]); + if (node == PieceTrie.DEAD) { + break; + } + keyPos++; + final int id = trie.value(node); + if (id < 0) { + continue; + } + if (unused[id]) { + continue; + } + maxFrontier = Math.max(maxFrontier, keyPos); + final int length = keyPos - startsAt; + // User-defined symbols receive a length bonus instead of a trained score. + final float score = userDefined[id] ? 0.1f * (length - 1) : scores[id]; + final float candidate = score + bestScoreTillHere; + if (bestStartsAt[keyPos] == -1 || candidate > bestScore[keyPos]) { + bestScore[keyPos] = candidate; + bestStartsAt[keyPos] = startsAt; + bestId[keyPos] = id; + } + if (!hasSingleNode && length == mblen) { + hasSingleNode = true; + } + } + + if (!hasSingleNode) { + final int end = startsAt + mblen; + maxFrontier = Math.max(maxFrontier, end); + final float candidate = unkScore + bestScoreTillHere; + if (bestStartsAt[end] == -1 || candidate > bestScore[end]) { + bestScore[end] = candidate; + bestStartsAt[end] = startsAt; + bestId[end] = unkId; + } + } + + startsAt += mblen; + } + + final List results = new ArrayList<>(size / 4 + 1); + int endsAt = size; + while (endsAt > 0) { + final int from = bestStartsAt[endsAt]; + if (from < 0) { + throw new IllegalStateException( + "The Viterbi path is broken at normalized byte " + endsAt + "."); + } + results.add(new Segment(from, endsAt, bestId[endsAt])); + endsAt = from; + } + Collections.reverse(results); + return results; + } +} diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java new file mode 100644 index 0000000000..e331fdcc83 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +/** + * A caller's text encoded as UTF-8, keeping the map from every byte offset back to the UTF-16 + * offset it came from. + * + *

The whole pipeline runs in byte space to match the reference implementation, but the spans + * reported to the caller must be UTF-16 offsets into the original {@code CharSequence}; this map + * converts them. An unpaired surrogate, which UTF-8 cannot represent, is encoded as U+FFFD, kept + * deterministic so that parity fixtures can cover it.

+ */ +final class Utf8Text { + + private final byte[] bytes; + private final int[] byteToChar; + private final int charLength; + + private Utf8Text(byte[] bytes, int[] byteToChar, int charLength) { + this.bytes = bytes; + this.byteToChar = byteToChar; + this.charLength = charLength; + } + + /** + * Encodes text. + * + * @param text The text to encode; must not be null. + * @return The encoded view. + */ + static Utf8Text of(CharSequence text) { + final int charLength = text.length(); + final byte[] bytes = new byte[charLength * 3 + 1]; + final int[] byteToChar = new int[charLength * 3 + 2]; + int b = 0; + int c = 0; + while (c < charLength) { + int codePoint = text.charAt(c); + int charCount = 1; + if (Character.isHighSurrogate((char) codePoint) && c + 1 < charLength + && Character.isLowSurrogate(text.charAt(c + 1))) { + codePoint = Character.toCodePoint((char) codePoint, text.charAt(c + 1)); + charCount = 2; + } else if (Character.isSurrogate((char) codePoint)) { + // An unpaired surrogate has no UTF-8 form; U+FFFD keeps the encoding total. + codePoint = 0xFFFD; + } + final int start = b; + if (codePoint < 0x80) { + bytes[b++] = (byte) codePoint; + } else if (codePoint < 0x800) { + bytes[b++] = (byte) (0xC0 | codePoint >>> 6); + bytes[b++] = (byte) (0x80 | codePoint & 0x3F); + } else if (codePoint < 0x10000) { + bytes[b++] = (byte) (0xE0 | codePoint >>> 12); + bytes[b++] = (byte) (0x80 | codePoint >>> 6 & 0x3F); + bytes[b++] = (byte) (0x80 | codePoint & 0x3F); + } else { + bytes[b++] = (byte) (0xF0 | codePoint >>> 18); + bytes[b++] = (byte) (0x80 | codePoint >>> 12 & 0x3F); + bytes[b++] = (byte) (0x80 | codePoint >>> 6 & 0x3F); + bytes[b++] = (byte) (0x80 | codePoint & 0x3F); + } + for (int i = start; i < b; i++) { + byteToChar[i] = c; + } + c += charCount; + } + byteToChar[b] = charLength; + final byte[] exact = java.util.Arrays.copyOf(bytes, b); + final int[] exactMap = java.util.Arrays.copyOf(byteToChar, b + 1); + return new Utf8Text(exact, exactMap, charLength); + } + + /** {@return the UTF-8 bytes} */ + byte[] bytes() { + return bytes; + } + + /** {@return the length of the original text in UTF-16 units} */ + int charLength() { + return charLength; + } + + /** + * Maps a byte offset to the UTF-16 offset of the character containing it. + * + * @param byteOffset An offset in {@code [0, bytes().length]}. + * @return The UTF-16 offset; the text length for the end offset. + */ + int charOffset(int byteOffset) { + return byteToChar[byteOffset]; + } +} diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java new file mode 100644 index 0000000000..02815aa372 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.subword.SubwordPiece; +import opennlp.tools.util.Span; +import opennlp.tools.util.normalizer.AlignedText; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the {@code OffsetAwareNormalizer} view: the model normalizer's output must align + * back to the original text exactly, including through whitespace collapsing, character-map + * replacements, and supplementary characters. + */ +class SentencePieceAlignmentTest { + + private static SentencePieceTokenizer unigram() { + return SentencePieceParityTest.tokenizer("tiny-unigram"); + } + + @ParameterizedTest + @ValueSource(strings = {"Hello world", " Hello world ", "Hello world.\nSecond line", + "3.14159 x 42", "family emoji", " leading and trailing ", "a", "", " "}) + void testAlignedMatchesUnaligned(String input) { + final AlignedText aligned = unigram().normalizeAligned(input); + assertEquals(unigram().normalize(input).toString(), aligned.normalizedString()); + assertEquals(input, aligned.original().toString()); + assertEquals(aligned.normalizedString().length(), aligned.alignment().normalizedLength()); + assertEquals(input.length(), aligned.alignment().originalLength()); + } + + @Test + void testWordMapsBackThroughCollapsedWhitespace() { + final String input = " Hello world "; + final AlignedText aligned = unigram().normalizeAligned(input); + final String normalized = aligned.normalizedString(); + + final int at = normalized.indexOf("world"); + final Span original = aligned.toOriginalSpan(at, at + "world".length()); + assertEquals("world", input.substring(original.getStart(), original.getEnd())); + } + + @Test + void testLigatureReplacementMapsToItsSourceCharacter() { + // The character map expands the single ligature to two letters; both normalized letters + // must map back to the one original character. + final String input = cp(0xFB01) + "nancial"; + final AlignedText aligned = unigram().normalizeAligned(input); + final String normalized = aligned.normalizedString(); + + final int at = normalized.indexOf("fi"); + assertTrue(at >= 0, "the character map must expand the ligature, got " + normalized); + final Span original = aligned.toOriginalSpan(at, at + 2); + assertEquals(0, original.getStart()); + assertEquals(1, original.getEnd()); + } + + @Test + void testSupplementaryCharacterSpansUseUtf16Units() { + final String input = "I love " + new String(Character.toChars(0x1F355)) + " pizza"; + final List pieces = unigram().encode(input); + + SubwordPiece pizzaSlice = null; + for (final SubwordPiece piece : pieces) { + if (piece.piece().contains(new String(Character.toChars(0x1F355)))) { + pizzaSlice = piece; + } + } + assertTrue(pizzaSlice != null, "the emoji must surface as a piece, got " + pieces); + assertEquals(7, pizzaSlice.start()); + assertEquals(9, pizzaSlice.end()); + assertEquals(new String(Character.toChars(0x1F355)), + input.substring(pizzaSlice.start(), pizzaSlice.end())); + } + + @Test + void testEverySpanIsWithinTheOriginalText() { + final String input = "quotes " + cp(0x201C) + "fancy" + cp(0x201D) + " and " + + cp(0x2018) + "single" + cp(0x2019) + " " + cp(0x2014) + " dash"; + for (final SubwordPiece piece : unigram().encode(input)) { + assertTrue(piece.start() >= 0 && piece.end() <= input.length(), + "span " + piece + " must lie inside the input"); + assertTrue(piece.start() <= piece.end(), "span " + piece + " must not be inverted"); + } + } + + @Test + void testSpansAreMonotonicAndAdjacent() { + final String input = "The quick brown fox jumps over the lazy dog."; + int previousEnd = 0; + for (final SubwordPiece piece : unigram().encode(input)) { + assertTrue(piece.start() >= previousEnd || piece.start() == piece.end(), + "piece " + piece + " must not step back before " + previousEnd); + previousEnd = Math.max(previousEnd, piece.end()); + } + assertEquals(input.length(), previousEnd, "the last span must reach the end of the input"); + } + + @Test + void testUnpairedSurrogateIsDeterministic() { + final String input = "a" + (char) 0xD83C + "b"; + final List first = unigram().encode(input); + final List second = unigram().encode(input); + assertEquals(first, second); + int covered = 0; + for (final SubwordPiece piece : first) { + covered = Math.max(covered, piece.end()); + } + assertEquals(input.length(), covered); + } + + private static String cp(int codePoint) { + return new String(Character.toChars(codePoint)); + } + + @Test + void testNullInputsFailLoudly() { + assertThrows(IllegalArgumentException.class, () -> unigram().encode(null)); + assertThrows(IllegalArgumentException.class, () -> unigram().normalizeAligned(null)); + assertThrows(IllegalArgumentException.class, () -> unigram().normalize(null)); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java new file mode 100644 index 0000000000..8f0b6c64a9 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Test; + +import opennlp.subword.SubwordPiece; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Fail-loud behavior on malformed models, plus the concurrency guarantee: one loaded tokenizer + * must produce identical results from many threads. + */ +class SentencePieceModelValidationTest { + + @Test + void testNullAndEmptyInputFailLoudly() { + assertThrows(IllegalArgumentException.class, + () -> SentencePieceTokenizer.load((java.nio.file.Path) null)); + assertThrows(IllegalArgumentException.class, + () -> SentencePieceTokenizer.load((InputStream) null)); + assertThrows(IllegalArgumentException.class, + () -> SentencePieceTokenizer.load(new ByteArrayInputStream(new byte[0]))); + } + + @Test + void testGarbageBytesFailLoudly() { + final byte[] garbage = "this is not a model file at all".getBytes(StandardCharsets.UTF_8); + assertThrows(IllegalArgumentException.class, + () -> SentencePieceTokenizer.load(new ByteArrayInputStream(garbage))); + } + + @Test + void testTruncatedModelFailsLoudly() throws IOException { + final byte[] whole = readModel(); + final byte[] truncated = java.util.Arrays.copyOf(whole, whole.length / 3); + assertThrows(IllegalArgumentException.class, + () -> SentencePieceTokenizer.load(new ByteArrayInputStream(truncated))); + } + + @Test + void testUnsupportedModelTypeFailsLoudly() { + // A minimal well-formed model claiming the WORD algorithm (model_type = 3). + final byte[] model = minimalModel(3); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> SentencePieceTokenizer.load(new ByteArrayInputStream(model))); + assertTrue(e.getMessage().contains("not supported"), e.getMessage()); + } + + @Test + void testMissingUnknownPieceFailsLoudly() { + final byte[] model = minimalModelWithoutUnk(); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> SentencePieceTokenizer.load(new ByteArrayInputStream(model))); + assertTrue(e.getMessage().contains("unknown piece"), e.getMessage()); + } + + @Test + void testConcurrentEncodingIsConsistent() throws Exception { + final SentencePieceTokenizer tokenizer = SentencePieceParityTest.tokenizer("tiny-unigram"); + final String[] inputs = { + "The quick brown fox jumps over the lazy dog.", + "tokenization and segmentation", + " Hello world ", + "water running walked faster apple book work play"}; + final List> expected = new ArrayList<>(); + for (final String input : inputs) { + expected.add(tokenizer.encode(input)); + } + + final ExecutorService pool = Executors.newFixedThreadPool(8); + try { + final List> futures = new ArrayList<>(); + for (int t = 0; t < 8; t++) { + futures.add(pool.submit((Callable) () -> { + for (int round = 0; round < 500; round++) { + for (int i = 0; i < inputs.length; i++) { + if (!expected.get(i).equals(tokenizer.encode(inputs[i]))) { + return false; + } + } + } + return true; + })); + } + for (final Future future : futures) { + assertTrue(future.get(), "concurrent encoding must match single-threaded results"); + } + } finally { + pool.shutdownNow(); + } + } + + @Test + void testVocabularyAccessors() { + final SentencePieceTokenizer tokenizer = SentencePieceParityTest.tokenizer("tiny-unigram"); + assertEquals(300, tokenizer.vocabularySize()); + assertEquals(SentencePieceTokenizer.Algorithm.UNIGRAM, tokenizer.algorithm()); + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + final String piece = tokenizer.idToPiece(id); + if (!tokenizer.isUnknown(id) && !tokenizer.isControl(id)) { + assertEquals(id, tokenizer.pieceToId(piece), "round trip of piece '" + piece + "'"); + } + } + assertEquals(tokenizer.unknownId(), tokenizer.pieceToId("definitely-not-in-the-vocabulary")); + assertThrows(IllegalArgumentException.class, () -> tokenizer.idToPiece(-1)); + assertThrows(IllegalArgumentException.class, + () -> tokenizer.idToPiece(tokenizer.vocabularySize())); + assertThrows(IllegalArgumentException.class, () -> tokenizer.pieceToId(null)); + } + + private static byte[] readModel() throws IOException { + try (InputStream in = + SentencePieceModelValidationTest.class.getResourceAsStream("tiny-unigram.model")) { + return in.readAllBytes(); + } + } + + // Hand-encodes a minimal ModelProto: three pieces (, , ) and a trainer spec with + // the requested model type. + private static byte[] minimalModel(int modelType) { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + writePiece(out, "", 2); + writePiece(out, "", 3); + writePiece(out, "", 3); + writePiece(out, "a", 1); + // trainer_spec { model_type = } + out.write(0x12); + out.write(2); + out.write(0x18); + out.write(modelType); + return out.toByteArray(); + } + + private static byte[] minimalModelWithoutUnk() { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + writePiece(out, "a", 1); + writePiece(out, "b", 1); + return out.toByteArray(); + } + + private static void writePiece(ByteArrayOutputStream out, String piece, int type) { + final byte[] utf8 = piece.getBytes(StandardCharsets.UTF_8); + // pieces { piece = ; score = 0.0; type = } as nested length-delimited field 1. + final int inner = 2 + utf8.length + 2; + out.write(0x0A); + out.write(inner); + out.write(0x0A); + out.write(utf8.length); + out.writeBytes(utf8); + out.write(0x18); + out.write(type); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java new file mode 100644 index 0000000000..4485e13f16 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; +import java.util.concurrent.ConcurrentHashMap; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.subword.SubwordPiece; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Asserts exact parity with the reference implementation: for every fixture input, the pieces, + * ids, original-text spans, and the normalized form must equal what the reference produced for + * the same bundled model. The fixtures were generated by {@code gen_fixtures.tsv}'s sibling + * script (see the test resources) against the sentencepiece Python package. + */ +class SentencePieceParityTest { + + private static final Map LOADED = new ConcurrentHashMap<>(); + + static SentencePieceTokenizer tokenizer(String model) { + return LOADED.computeIfAbsent(model, name -> { + try (InputStream in = SentencePieceParityTest.class.getResourceAsStream(name + ".model")) { + assertNotNull(in, "missing test resource " + name + ".model"); + return SentencePieceTokenizer.load(in); + } catch (IOException e) { + throw new IllegalStateException(e); + } + }); + } + + @ParameterizedTest + @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", + "tiny-unigram-identity", "tiny-unigram-suffix"}) + void testFixtureParity(String model) throws IOException { + final SentencePieceTokenizer tokenizer = tokenizer(model); + int lines = 0; + for (final Fixture fixture : fixtures(model)) { + lines++; + final List actual = tokenizer.encode(fixture.input); + final String context = model + " input <" + fixture.input + ">"; + assertEquals(fixture.pieces.size(), actual.size(), + context + " piece count; got " + actual); + for (int i = 0; i < actual.size(); i++) { + final SubwordPiece expected = fixture.pieces.get(i); + final SubwordPiece got = actual.get(i); + assertEquals(expected.piece(), got.piece(), context + " piece " + i); + assertEquals(expected.id(), got.id(), context + " id of piece " + i); + assertEquals(expected.start(), got.start(), context + " start of piece " + i); + assertEquals(expected.end(), got.end(), context + " end of piece " + i); + } + assertEquals(fixture.normalized, tokenizer.normalize(fixture.input).toString(), + context + " normalized form"); + } + assertTrue(lines >= 30, "the fixture file must not be empty or truncated"); + } + + @ParameterizedTest + @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", + "tiny-unigram-identity", "tiny-unigram-suffix"}) + void testEmbeddedSelfTestSamples(String model) { + final SentencePieceTokenizer tokenizer = tokenizer(model); + final List inputs = tokenizer.selfTestInputs(); + final List expected = tokenizer.selfTestExpected(); + assertTrue(!inputs.isEmpty(), "the tiny models embed self-test samples"); + for (int i = 0; i < inputs.size(); i++) { + final StringJoiner joined = new StringJoiner(" "); + for (final String piece : tokenizer.encodeToPieces(inputs.get(i))) { + joined.add(piece); + } + assertEquals(expected.get(i), joined.toString(), + model + " self-test sample <" + inputs.get(i) + ">"); + } + } + + private record Fixture(String input, List pieces, String normalized) { + } + + private static List fixtures(String model) throws IOException { + final List fixtures = new ArrayList<>(); + try (InputStream in = + SentencePieceParityTest.class.getResourceAsStream(model + ".fixtures.tsv")) { + assertNotNull(in, "missing test resource " + model + ".fixtures.tsv"); + final BufferedReader reader = + new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + String line; + while ((line = reader.readLine()) != null) { + final String[] cols = line.split("\t", -1); + final String input = unescape(cols[0]); + final int count = Integer.parseInt(cols[1]); + final List pieces = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + pieces.add(new SubwordPiece(unescape(cols[2 + i * 4]), + Integer.parseInt(cols[3 + i * 4]), Integer.parseInt(cols[4 + i * 4]), + Integer.parseInt(cols[5 + i * 4]))); + } + fixtures.add(new Fixture(input, pieces, unescape(cols[2 + count * 4]))); + } + } + return fixtures; + } + + private static String unescape(String s) { + final StringBuilder out = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + if (c == '\\' && i + 1 < s.length()) { + i++; + switch (s.charAt(i)) { + case 't' -> out.append('\t'); + case 'n' -> out.append('\n'); + case 'r' -> out.append('\r'); + case '\\' -> out.append('\\'); + default -> throw new IllegalArgumentException("bad escape in fixture: " + s); + } + } else { + out.append(c); + } + } + return out.toString(); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java new file mode 100644 index 0000000000..e4acccd499 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; + +import opennlp.subword.SubwordPiece; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Opt-in parity check against real pre-trained models, which are downloaded rather than bundled. + * + *

Point {@code -Dopennlp.subword.eval.dir} at a directory holding {@code .model} files + * with sibling {@code .fixtures.tsv} files generated by the {@code gen_real_fixtures.py} + * script from the test resources; every model found is asserted piece for piece. Without the + * property the test is skipped.

+ */ +class SentencePieceRealModelEvalTest { + + @Test + void testRealModelParity() throws IOException { + final String dir = System.getProperty("opennlp.subword.eval.dir"); + assumeTrue(dir != null && !dir.isBlank(), + "set -Dopennlp.subword.eval.dir to run the real-model parity check"); + + int models = 0; + try (Stream files = Files.list(Path.of(dir))) { + for (final Path model : files.filter(f -> f.toString().endsWith(".model")).sorted() + .toList()) { + final Path fixtures = Path.of(model.toString() + .substring(0, model.toString().length() - ".model".length()) + ".fixtures.tsv"); + assumeTrue(Files.exists(fixtures), "no fixtures for " + model.getFileName()); + models++; + assertModel(model, fixtures); + } + } + assertTrue(models > 0, "the eval directory contains no models"); + } + + private static void assertModel(Path modelPath, Path fixturesPath) throws IOException { + final SentencePieceTokenizer tokenizer = SentencePieceTokenizer.load(modelPath); + int lines = 0; + try (BufferedReader reader = Files.newBufferedReader(fixturesPath, StandardCharsets.UTF_8)) { + String line; + while ((line = reader.readLine()) != null) { + lines++; + final String[] cols = line.split("\t", -1); + final String input = unescape(cols[0]); + final int count = Integer.parseInt(cols[1]); + final String context = modelPath.getFileName() + " input <" + input + ">"; + + final List actual = tokenizer.encode(input); + assertEquals(count, actual.size(), context + " piece count; got " + actual); + for (int i = 0; i < count; i++) { + final SubwordPiece got = actual.get(i); + assertEquals(unescape(cols[2 + i * 4]), got.piece(), context + " piece " + i); + assertEquals(Integer.parseInt(cols[3 + i * 4]), got.id(), context + " id " + i); + assertEquals(Integer.parseInt(cols[4 + i * 4]), got.start(), context + " start " + i); + assertEquals(Integer.parseInt(cols[5 + i * 4]), got.end(), context + " end " + i); + } + assertEquals(unescape(cols[2 + count * 4]), tokenizer.normalize(input).toString(), + context + " normalized form"); + } + } + assertTrue(lines >= 30, modelPath.getFileName() + " fixtures must not be truncated"); + } + + private static String unescape(String s) { + final StringBuilder out = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + if (c == '\\' && i + 1 < s.length()) { + i++; + switch (s.charAt(i)) { + case 't' -> out.append('\t'); + case 'n' -> out.append('\n'); + case 'r' -> out.append('\r'); + case '\\' -> out.append('\\'); + default -> throw new IllegalArgumentException("bad escape in fixture: " + s); + } + } else { + out.append(c); + } + } + return out.toString(); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/corpus.txt b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/corpus.txt new file mode 100644 index 0000000000..d578654315 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/corpus.txt @@ -0,0 +1,66 @@ +The quick brown fox jumps over the lazy dog. +Apache OpenNLP is a machine learning based toolkit for the processing of natural language text. +It supports the most common NLP tasks, such as tokenization, sentence segmentation, and named entity extraction. +Subword tokenization decomposes words into smaller units drawn from a fixed vocabulary. +The unigram language model selects the segmentation with the highest total log probability. +Byte pair encoding merges the most frequent adjacent symbol pairs until no merge applies. +A sentence piece model carries its own text normalizer inside the model file. +Character offsets should always point back into the original text. +Whitespace is escaped with a special marker so word boundaries survive segmentation. +Numbers like 3.14159 and 42 and 1024 appear in ordinary text. +Punctuation, quotes, and dashes are folded by the normalizer! +Questions? Answers! Ellipses... and (parentheses) too. +The cafe served naive patrons a souffle with creme fraiche. +Internationalization and localization are long words. +Antidisestablishmentarianism remains one of the longest English words. +She sells seashells by the seashore. +Peter Piper picked a peck of pickled peppers. +How much wood would a woodchuck chuck if a woodchuck could chuck wood? +The rain in Spain stays mainly in the plain. +To be or not to be, that is the question. +All happy families are alike; each unhappy family is unhappy in its own way. +It was the best of times, it was the worst of times. +Call me Ishmael. +In the beginning was the word. +The world is everything that is the case. +Language models assign probabilities to sequences of tokens. +Retrieval systems rank documents by similarity to a query. +Embeddings map text into dense vector spaces. +Search engines combine lexical and semantic signals. +The tokenizer must be fast, deterministic, and thread safe. +Model files are loaded once and shared across threads. +Tests must prove parity with the reference implementation. +Offsets are measured in code units of the original encoding. +The normalizer collapses repeated whitespace into one marker. +A leading marker separates words that start a sentence. +Unknown characters fall back to a penalty score. +Byte fallback decomposes unknown characters into byte pieces. +User defined symbols are never split by the tokenizer. +Control symbols never appear in encoded output. +The vocabulary maps each piece to an integer identifier. +Scores are log probabilities in the unigram model. +Merge ranks order the byte pair encoding agenda. +The trie enumerates every piece that starts at a position. +Dynamic programming finds the best path in one pass. +Backtracking recovers the winning segmentation. +The agenda is a priority queue ordered by score. +Stale entries are skipped when their symbols have merged. +Un texto corto en espanol para variar el corpus. +Un petit texte en francais pour la diversite. +Ein kurzer deutscher Satz steht auch hier. +Ancora una frase italiana per completezza. +Tokenization quality depends on the training corpus. +The model was trained on a tiny corpus for testing only. +Nothing in this file is quoted from any external work. +Short lines help. +One. +Two words. +Three little words. +water water water water water +running runner ran runs +walked walking walker walks +faster fastest fast +apple apples applesauce +book books bookshelf bookstore +work works worked working worker +play plays played playing player diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_fixtures.py b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_fixtures.py new file mode 100644 index 0000000000..658902e48f --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_fixtures.py @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Trains the tiny SentencePiece test models and generates the parity fixtures. + +Run inside a venv with the sentencepiece package installed: + python gen_fixtures.py + +Every fixture line is tab-separated with backslash escaping (\\\\, \\t, \\n, \\r): + esc(input) TAB pieceCount TAB [esc(piece) TAB id TAB begin TAB end]... TAB esc(normalized) +Offsets are UTF-16 code-unit offsets into the original input, matching Java string indexing. +""" +import sys +import sentencepiece as spm + +MULTILINGUAL = [ + "Le café coûte trois euros à Paris.", + "Der Straßenname ändert sich häufig.", + "Ça va très bien, merci beaucoup.", + "El niño pequeño come una manzana.", + "Привет мир и всем добро.", + "東京タワーに登りました。", + "日本語の文章も少しあります。", + "안녕하세요 세계입니다.", + "你好世界这是中文。", + "I love \U0001f355 and \U0001f1e9\U0001f1ea a lot!", + "Emoji test \U0001f600 \U0001f680 ❤️ done.", +] + +INPUTS = [ + "", + " ", + " ", + "a", + "Hello world", + " Hello world ", + "Hello world.\nSecond line\ttabbed", + "The quick brown fox jumps over the lazy dog.", + "tokenization and segmentation", + "Antidisestablishmentarianism", + "water running walked faster apple book work play", + "3.14159 x 42 = 1024?", + "!!!???...", + "(parentheses) and [brackets] and {braces}", + "café naïve fiancé résumé", + "financial fluid", + "① ⑪ ㋿ ＫＡＴＡＫＡＮＡ", + "ｶﾀｶﾅ half width", + "東京タワーへ行きました", + "日本語とEnglish混在", + "Привет мир", + "안녕하세요 세계", + "你好，世界！", + "I love \U0001f355 pizza", + "flags \U0001f1e9\U0001f1ea \U0001f1fa\U0001f1f8 end", + "family \U0001f469‍\U0001f469‍\U0001f467‍\U0001f466 emoji", + "zero​width and non breaking", + "quotes “fancy” and ‘single’ — dash", + " the [URL] token", + "a b[URL]c", + "control tokens inline", + "https://example.com/path?q=1&x=2", + "UPPER lower MiXeD case", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "Ω≈ç√∫˜µ≤", + "مرحبا بالعالم", + " leading and trailing ", + "\ttab\tstart", + "newline\n\n\nruns", + "mid spaces collapse", +] + +MODELS = { + "tiny-unigram": dict(model_type="unigram", vocab_size=300), + "tiny-unigram-bytefb": dict(model_type="unigram", vocab_size=600, byte_fallback=True, + character_coverage=0.995), + "tiny-bpe": dict(model_type="bpe", vocab_size=300), + "tiny-unigram-identity": dict(model_type="unigram", vocab_size=300, + normalization_rule_name="identity"), + "tiny-unigram-suffix": dict(model_type="unigram", vocab_size=300, + treat_whitespace_as_suffix=True), +} + + +def esc(s): + return (s.replace("\\", "\\\\").replace("\t", "\\t") + .replace("\n", "\\n").replace("\r", "\\r")) + + +def utf16_offset(text, codepoint_offset): + return len(text[:codepoint_offset].encode("utf-16-le")) // 2 + + +def main(corpus, outdir): + full_corpus = outdir + "/corpus-full.txt" + with open(corpus, encoding="utf-8") as f: + lines = f.read().splitlines() + lines += MULTILINGUAL + with open(full_corpus, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + for name, opts in MODELS.items(): + spm.SentencePieceTrainer.Train( + input=full_corpus, + model_prefix=outdir + "/" + name, + hard_vocab_limit=False, + character_coverage=opts.pop("character_coverage", 1.0), + user_defined_symbols=["", "[URL]"], + self_test_sample_size=10, + **opts, + ) + sp = spm.SentencePieceProcessor(model_file=outdir + "/" + name + ".model") + with open(outdir + "/" + name + ".fixtures.tsv", "w", encoding="utf-8") as out: + for text in INPUTS: + proto = sp.EncodeAsImmutableProto(text) + cols = [esc(text), str(len(proto.pieces))] + for piece in proto.pieces: + cols += [esc(piece.piece), str(piece.id), + str(utf16_offset(text, piece.begin)), + str(utf16_offset(text, piece.end))] + cols.append(esc(sp.Normalize(text))) + out.write("\t".join(cols) + "\n") + print(name, "vocab", sp.GetPieceSize()) + + +if __name__ == "__main__": + main(sys.argv[1], sys.argv[2]) diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_real_fixtures.py b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_real_fixtures.py new file mode 100644 index 0000000000..9b27c7a11e --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/gen_real_fixtures.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generates parity fixtures for pre-trained real-world models (no training). + +Usage: python gen_real_fixtures.py +Reads every *.model in the directory and writes a sibling *.fixtures.tsv in the same +escaped-TSV format as gen_fixtures.py, over a larger and messier input list. +""" +import glob +import os +import sys +import sentencepiece as spm +from gen_fixtures import INPUTS, esc, utf16_offset + +EXTRA = [ + "The Transformer architecture revolutionized natural language processing in 2017.", + "supercalifragilisticexpialidocious and pneumonoultramicroscopicsilicovolcanoconiosis", + "e=mc^2, F=ma, and a^2+b^2=c^2 are famous equations.", + "Mixed scripts: English, 日本語, 한국어, русский, and العربية together.", + "Prices: $19.99, €25,50, £12, ¥1500, and ₹999.", + "C++ and C# and F# are programming languages; so is Java.", + "def encode(text): return sp.encode(text, out_type=str)", + "SELECT * FROM documents WHERE score > 0.5 ORDER BY rank;", + "The 2024 Summer Olympics were held in Paris, France.", + "COVID-19 vaccines use mRNA technology (Pfizer-BioNTech, Moderna).", + "Email me at test.user+tag@example.co.uk or call +1 (555) 010-9999.", + "10,000 steps a day keeps the doctor away... allegedly!", + "The naive resume of the fiancee included a cafe visit.", + "¿Dónde está la biblioteca? ¡Allí está!", + "Smørrebrød og æbleskiver er danske specialiteter.", + "Zażółć gęślą jaźń is a Polish pangram.", + "Đây là tiếng Việt với nhiều dấu.", + "今日はいい天気ですね。明日も晴れるといいな。", + "北京和上海都是大城市。", + "한국의 수도는 서울입니다.", + "\U0001f9d1‍\U0001f4bb codes while \U0001f9d1‍\U0001f373 cooks \U0001f35c!", + "
line separator and 
paragraph separator lurk here", + "BOM at the start of this sentence", + "tabs\tand\ttabs\tand\ttabs", + "CRLF\r\nline endings\r\nhappen", +] + + +def main(model_dir): + for model_path in sorted(glob.glob(os.path.join(model_dir, "*.model"))): + name = os.path.splitext(model_path)[0] + sp = spm.SentencePieceProcessor(model_file=model_path) + with open(name + ".fixtures.tsv", "w", encoding="utf-8") as out: + for text in INPUTS + EXTRA: + proto = sp.EncodeAsImmutableProto(text) + cols = [esc(text), str(len(proto.pieces))] + for piece in proto.pieces: + cols += [esc(piece.piece), str(piece.id), + str(utf16_offset(text, piece.begin)), + str(utf16_offset(text, piece.end))] + cols.append(esc(sp.Normalize(text))) + out.write("\t".join(cols) + "\n") + print(os.path.basename(name), "vocab", sp.GetPieceSize()) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.fixtures.tsv new file mode 100644 index 0000000000..e3560148f9 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.fixtures.tsv @@ -0,0 +1,40 @@ + 0 + 0 + 0 +a 1 ▁a 8 0 1 ▁a +Hello world 7 ▁ 177 0 0 H 246 0 1 el 48 1 3 l 186 3 4 o 183 4 5 ▁wor 38 5 9 ld 129 9 11 ▁Hello▁world + Hello world 7 ▁ 177 1 1 H 246 1 2 el 48 2 4 l 186 4 5 o 183 5 6 ▁wor 38 6 12 ld 129 12 14 ▁Hello▁world +Hello world.\nSecond line\ttabbed 19 ▁ 177 0 0 H 246 0 1 el 48 1 3 l 186 3 4 o 183 4 5 ▁wor 38 5 9 ld 129 9 11 . 193 11 12 ▁S 64 12 14 ec 51 14 16 on 16 16 18 d 189 18 19 ▁l 28 19 21 in 7 21 23 e 178 23 24 ▁t 5 24 26 ab 85 26 28 b 199 28 29 ed 24 29 31 ▁Hello▁world.▁Second▁line▁tabbed +The quick brown fox jumps over the lazy dog. 27 ▁The 50 0 3 ▁qu 81 3 6 ic 61 6 8 k 196 8 9 ▁b 23 9 11 ro 41 11 13 wn 90 13 15 ▁f 25 15 17 o 183 17 18 x 203 18 19 ▁ 177 19 20 j 220 20 21 u 192 21 22 mp 108 22 24 s 182 24 25 ▁o 45 25 27 v 200 27 28 er 6 28 30 ▁the 19 30 34 ▁l 28 34 36 a 179 36 37 z 202 37 38 y 198 38 39 ▁d 42 39 41 o 183 41 42 g 194 42 43 . 193 43 44 ▁The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog. +tokenization and segmentation 4 ▁tokeniz 170 0 7 ation 47 7 12 ▁and 59 12 16 ▁segmentation 171 16 29 ▁tokenization▁and▁segmentation +Antidisestablishmentarianism 16 ▁A 76 0 1 n 181 1 2 t 180 2 3 id 176 3 5 is 31 5 7 est 57 7 10 ab 85 10 12 l 186 12 13 is 31 13 15 h 187 15 16 ment 83 16 20 ar 18 20 22 i 184 22 23 an 40 23 25 is 31 25 27 m 191 27 28 ▁Antidisestablishmentarianism +water running walked faster apple book work play 15 ▁water 124 0 5 ▁r 93 5 7 un 39 7 9 ning 154 9 13 ▁wal 160 13 17 k 196 17 18 ed 24 18 20 ▁fast 162 20 25 er 6 25 27 ▁app 99 27 31 le 107 31 33 ▁boo 155 33 37 k 196 37 38 ▁work 167 38 43 ▁play 123 43 48 ▁water▁running▁walked▁faster▁apple▁book▁work▁play +3.14159 x 42 = 1024? 21 ▁ 177 0 0 3 242 0 1 . 193 1 2 1 215 2 3 4 216 3 4 1 215 4 5 5 243 5 6 9 244 6 7 ▁ 177 7 8 x 203 8 9 ▁ 177 9 10 4 216 10 11 2 224 11 12 ▁ 177 12 13 = 0 13 14 ▁ 177 14 15 1 215 15 16 0 241 16 17 2 224 17 18 4 216 18 19 ? 225 19 20 ▁3.14159▁x▁42▁=▁1024? +!!!???... 10 ▁ 177 0 0 ! 214 0 1 ! 214 1 2 ! 214 2 3 ? 225 3 4 ? 225 4 5 ? 225 5 6 . 193 6 7 . 193 7 8 . 193 8 9 ▁!!!???... +(parentheses) and [brackets] and {braces} 25 ▁ 177 0 0 ( 239 0 1 p 190 1 2 ar 18 2 4 ent 35 4 7 he 9 7 9 ses 114 9 12 ) 240 12 13 ▁and 59 13 17 ▁ 177 17 18 [ 0 18 19 b 199 19 20 r 185 20 21 ack 112 21 24 et 86 24 26 s 182 26 27 ] 0 27 28 ▁and 59 28 32 ▁ 177 32 33 { 0 33 34 b 199 34 35 r 185 35 36 ac 29 36 38 es 11 38 40 } 0 40 41 ▁(parentheses)▁and▁[brackets]▁and▁{braces} +café naïve fiancé résumé 18 ▁c 26 0 1 af 173 1 3 é 254 3 4 ▁n 49 4 6 a 179 6 7 ï 0 7 8 ve 109 8 10 ▁f 25 10 12 i 184 12 13 an 40 13 15 c 188 15 16 é 254 16 17 ▁r 93 17 19 é 254 19 20 s 182 20 21 u 192 21 22 m 191 22 23 é 254 23 24 ▁café▁naïve▁fiancé▁résumé +financial fluid 10 ▁f 25 0 0 in 7 0 2 an 40 2 4 c 188 4 5 i 184 5 6 al 20 6 8 ▁f 25 8 9 l 186 9 10 u 192 10 11 id 176 11 13 ▁financial▁fluid +① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 16 ▁ 177 0 0 1 215 0 1 ▁ 177 1 2 1 215 2 2 1 215 2 3 ▁ 177 3 4 令和 0 4 5 ▁ 177 5 6 K 0 6 7 A 207 7 8 T 201 8 9 A 207 9 10 K 0 10 11 A 207 11 12 N 212 12 13 A 207 13 14 ▁1▁11▁令和▁KATAKANA +ｶﾀｶﾅ half width 11 ▁ 177 0 0 カ 0 0 1 タ 268 1 2 カナ 0 2 4 ▁h 110 4 6 al 20 6 8 f 197 8 9 ▁w 14 9 11 id 176 11 13 t 180 13 14 h 187 14 15 ▁カタカナ▁half▁width +東京タワーへ行きました 10 ▁ 177 0 0 東 280 0 1 京 273 1 2 タ 268 2 3 ワ 269 3 4 ー 270 4 5 へ行き 0 5 8 ま 235 8 9 し 234 9 10 た 264 10 11 ▁東京タワーへ行きました +日本語とEnglish混在 12 ▁ 177 0 0 日 277 0 1 本 279 1 2 語 284 2 3 と 0 3 4 E 208 4 5 n 181 5 6 g 194 6 7 l 186 7 8 is 31 8 10 h 187 10 11 混在 0 11 13 ▁日本語とEnglish混在 +Привет мир 11 ▁ 177 0 0 П 256 0 1 р 222 1 2 и 221 2 3 в 230 3 4 е 231 4 5 т 260 5 6 ▁ 177 6 7 м 232 7 8 и 221 8 9 р 222 9 10 ▁Привет▁мир +안녕하세요 세계 9 ▁ 177 0 0 안 290 0 1 녕 287 1 2 하 293 2 3 세 238 3 4 요 291 4 5 ▁ 177 5 6 세 238 6 7 계 286 7 8 ▁안녕하세요▁세계 +你好，世界！ 7 ▁ 177 0 0 你 274 0 1 好 275 1 2 , 204 2 3 世 271 3 4 界 281 4 5 ! 214 5 6 ▁你好,世界! +I love 🍕 pizza 9 ▁I 92 0 1 ▁lo 96 1 4 ve 109 4 6 ▁ 177 6 7 🍕 297 7 9 ▁p 15 9 11 iz 54 11 13 z 202 13 14 a 179 14 15 ▁I▁love▁🍕▁pizza +flags 🇩🇪 🇺🇸 end 12 ▁f 25 0 1 l 186 1 2 a 179 2 3 g 194 3 4 s 182 4 5 ▁ 177 5 6 🇩 295 6 8 🇪 296 8 10 ▁ 177 10 11 🇺🇸 0 11 15 ▁en 149 15 18 d 189 18 19 ▁flags▁🇩🇪▁🇺🇸▁end +family 👩‍👩‍👧‍👦 emoji 12 ▁f 25 0 1 am 104 1 3 il 53 3 5 y 198 5 6 ▁ 177 6 7 👩‍👩‍👧‍👦 0 7 18 ▁ 177 18 19 e 178 19 20 m 191 20 21 o 183 21 22 j 220 22 23 i 184 23 24 ▁family▁👩‍👩‍👧‍👦▁emoji +zero​width and non breaking 16 ▁ 177 0 0 z 202 0 1 er 6 1 3 o 183 3 4 ▁w 14 4 6 id 176 6 8 t 180 8 9 h 187 9 10 ▁and 59 10 14 ▁n 49 14 16 on 16 16 18 ▁b 23 18 20 re 32 20 22 a 179 22 23 k 196 23 24 ing 27 24 27 ▁zero▁width▁and▁non▁breaking +quotes “fancy” and ‘single’ — dash 22 ▁qu 81 0 2 ot 130 2 4 es 11 4 6 ▁ 177 6 7 “ 0 7 8 f 197 8 9 an 40 9 11 c 188 11 12 y 198 12 13 ” 0 13 14 ▁and 59 14 18 ▁ 177 18 19 ‘ 0 19 20 s 182 20 21 ing 27 21 24 le 107 24 26 ’ 0 26 27 ▁ 177 27 28 — 0 28 29 ▁d 42 29 31 as 30 31 33 h 187 33 34 ▁quotes▁“fancy”▁and▁‘single’▁—▁dash + the [URL] token 7 ▁ 177 0 0 3 0 6 ▁the 19 6 10 ▁ 177 10 11 [URL] 4 11 16 ▁to 43 16 19 ken 95 19 22 ▁▁the▁[URL]▁token +a b[URL]c 6 ▁a 8 0 1 ▁ 177 1 2 3 2 8 b 199 8 9 [URL] 4 9 14 c 188 14 15 ▁a▁b[URL]c +control tokens inline 20 ▁c 26 0 1 on 16 1 3 t 180 3 4 ro 41 4 6 l 186 6 7 ▁ 177 7 8 < 0 8 9 s 182 9 10 > 0 10 11 ▁to 43 11 14 ken 95 14 17 s 182 17 18 ▁ 177 18 19 0 22 23 ▁in 37 23 26 l 186 26 27 in 7 27 29 e 178 29 30 ▁control▁▁tokens▁▁inline +https://example.com/path?q=1&x=2 26 ▁h 110 0 1 t 180 1 2 t 180 2 3 p 190 3 4 s 182 4 5 :// 0 5 8 ex 52 8 10 am 104 10 12 p 190 12 13 le 107 13 15 . 193 15 16 c 188 16 17 o 183 17 18 m 191 18 19 / 0 19 20 p 190 20 21 at 17 21 23 h 187 23 24 ? 225 24 25 q 205 25 26 = 0 26 27 1 215 27 28 & 0 28 29 x 203 29 30 = 0 30 31 2 224 31 32 ▁https://example.com/path?q=1&x=2 +UPPER lower MiXeD case 17 ▁U 135 0 1 P 210 1 2 P 210 2 3 E 208 3 4 R 248 4 5 ▁lo 96 5 8 w 195 8 9 er 6 9 11 ▁ 177 11 12 M 227 12 13 i 184 13 14 X 0 14 15 e 178 15 16 D 226 16 17 ▁c 26 17 19 as 30 19 21 e 178 21 22 ▁UPPER▁lower▁MiXeD▁case +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 8 0 1 a 179 1 2 a 179 2 3 a 179 3 4 a 179 4 5 a 179 5 6 a 179 6 7 a 179 7 8 a 179 8 9 a 179 9 10 a 179 10 11 a 179 11 12 a 179 12 13 a 179 13 14 a 179 14 15 a 179 15 16 a 179 16 17 a 179 17 18 a 179 18 19 a 179 19 20 a 179 20 21 a 179 21 22 a 179 22 23 a 179 23 24 a 179 24 25 a 179 25 26 a 179 26 27 a 179 27 28 a 179 28 29 a 179 29 30 a 179 30 31 a 179 31 32 a 179 32 33 a 179 33 34 a 179 34 35 a 179 35 36 a 179 36 37 a 179 37 38 a 179 38 39 a 179 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Ω≈ç√∫˜µ≤ 4 ▁ 177 0 0 Ω≈ç√∫ 0 0 5 ▁ 177 5 5 ̃μ≤ 0 5 8 ▁Ω≈ç√∫▁̃μ≤ +مرحبا بالعالم 4 ▁ 177 0 0 مرحبا 0 0 5 ▁ 177 5 6 بالعالم 0 6 13 ▁مرحبا▁بالعالم + leading and trailing 9 ▁l 28 2 3 e 178 3 4 ad 172 4 6 ing 27 6 9 ▁and 59 9 13 ▁t 5 13 15 ra 62 15 17 il 53 17 19 ing 27 19 22 ▁leading▁and▁trailing +\ttab\tstart 6 ▁t 5 1 2 ab 85 2 4 ▁s 13 4 6 t 180 6 7 ar 18 7 9 t 180 9 10 ▁tab▁start +newline\n\n\nruns 9 ▁n 49 0 1 e 178 1 2 w 195 2 3 l 186 3 4 in 7 4 6 e 178 6 7 ▁r 93 7 11 un 39 11 13 s 182 13 14 ▁newline▁runs +mid spaces collapse 11 ▁m 22 0 1 id 176 1 3 ▁s 13 3 7 pac 147 7 10 es 11 10 12 ▁c 26 12 16 ol 69 16 18 l 186 18 19 ap 127 19 21 s 182 21 22 e 178 22 23 ▁mid▁spaces▁collapse diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-bpe.model new file mode 100644 index 0000000000000000000000000000000000000000..8b6f22eb821254525c4707b8f902bab40f7356aa GIT binary patch literal 245063 zcmZU+3s_WFnyCF%6&G)Swz1JDqGKFm8{0U>QAaMZjjgt!jcpiX8;_xtRz?ZhNE_P_ zs<;(IZ9AV^yoiIL2c$$`D&^^`K)M;~0%#t_(y}{&K88l^r;S4>-2sm~c!bnW55oUQ<(4SzZjPDxFu3 zQKv9fMlIhrf={;aWUHp8sqn^uB-3#e4LE*<<2@}<>IT#g2mF7HZ}y%Azfpn+o%3P*t`0BABsK&uHpzREvD2sTnG#D+6h~isQPLcpjfYJt^YzWm}-x?LG>-v zI89AckzWj|ze9aVQ@3TIF45HOs+C^;Of_|CrfQ8>1`4Fr^VNZ@Qr=W`dQFS7K{fL0 z!9-QAZ)oao)L`78saeW*(?F3|;@A?*;Z5~m>p%{fD(~%DjFg?IZv~T79sahaW~kB| zgHhj7h07g?nx=}7Kj6wxgQxKOqG~-~q@}P~g?m?1)78k^Gf=1nIDQ{Y`8O)edzzk> zs`U2<(*2bx)xH6j)+ZkfxU@F>aKN>F6Cpp+5>F3~s)L#%V}sUxA8RE@Q6)I6si~?q zf6&wn)ft~?s@4HjnyL+&8cp4y`n+zS;EQn74>*?MXc%xP!)y#zW4kK&QBB{hO$W^b z1>K?sUCV$=8`fBXzE5a0RBZ&wjN#4_NLZne;IIZ-J%WZTi-WGN-+BW{%pwbA%oQ@8L;y^ zT3u7p$ml0g#`62Y9NFLb!|+BY5IC?xP)tZYKqEXkfv|a^ud~* zma0sANKn_TVrajv3O7vCGqm|QLesaVsPK`Ro~3nUR9FAJsSVuW11`;|M`)EueRI9) z%O32iSl&xfHPf7b zGgZw>PY*bEq^K4e`&Xx`+_7+JNqmCOxKVRLxw2x%orG-le>oea7dI2l)CZ1 z54ykYQj1xN8dFn(s+vTf)znlqrlxA@My38pQ`1z~XZ!4gJ5@G}Se8G)+Hy@(*7}O3O+;IvX*1D&p=mFmt<^8a;E&wUqtqo)7)12g(2O}~||3ZJHx<5{$S(zK~) zn>Fouv@M!83oTvKW~05SY4gxBG;KcGR!v)iwoTJkp#4VEeu0*$X{*q-YlZGog~$r3 zhfseTR1c%>2&y%xZw1v_)SW@qjru!HJ)@%S3aYBL-VUnGsM(r&R$0_LK~;4|j#lM= zuT-sU3^t1if8&6@AAeK7KWQ5WbQ}J;{r(O3_v^VT&Agz0t`GlyJb)Ra7J(~YtocZ_%^_vCd z`0v&HZ|*L_e?V95`FqX(R=M);*FV~ae@W2);{yDH`t9-$@qbVA?`tc;AD!dbokM2T z&7}j5I-U$T_NuJk?|Fx_+>5$jFSW?K4<`wq|8c|8>@uEkEA=*7hF^ zQ=Z$?-rPC;S6fp@>nGIfpC6J;?+W5V*vBH{A2RdI9W3i8=lT~TALY`A7M_!soZH=i|QzeF3s~ra?;4{a>*N zk>(C)9V^CU+QyhlS`nlf&ukK75dUz!1gKZ;RKjmJh{Y5pKEfKCC<)^~K4w>s6gZ@yy(#qky<0sPj%{X3Atj8q0O!pR1F=Q##pmOeY2S z`zVWLluo+#MayRgpOWMG(en8#(Q@KewEVO22Vy3_Qo@~L9R16c9}@S6vY0kM-8n)2 zcyNrI^*DwtXMV0M#xa=l9 z|FqN%lzmqUCkU2d6SoQV8Fo{7L&xeWWSeVuxfsjziT|phQ z!o_IEzdcL}XNQYI06IgW0@#=k)BNfL1rznXD!4A~SP zCT7wtEa!44hB6h4oWMV~mcBVJV%LSsIozd;&D{MuIYk7yM!rL21x8!IW(YB+(P@jl~&aJeR(8KIi%u|uLJ4-j_{xAq!rz#{0j^+iM&l$4e~(cal;^g zRBdTB$O)bssIwjSX%%)qw%l!yJs#Tr8Ok;Rls!L>?J}2|84Jc_+IF?bW75i_etQ@P zmx+4;&XSHPn=sq3rwOFdi2Y|9qH(9#(ySU{LHgY&`XO6_i~s7LKyE`k=v0khRcEJQR1%&mq2~E6w;UGC6U%GC{%Jlsb5&cxi&Lb*!l<2|>zvq8T=9d=W$qVy;!ChlnB{IiNTnZ2><2ihwY z85PSs1F;YdQ{V}4jZ>);L=q-EhTYdr)>nJ%*-g$eYWDtI_ zDqqIg6yy}b%|OmVnkq?iBJJT)?L+?8$+LMF^X@RI3ecy=S=R_#Vv3SI#ZjW{b{_HO z;I|0bbdtEFyBv2SnCYig$UO2)pq zodN!O#@QF#)lesMnn*9w zUA0svJ1Bc2WwNYhZK!2Tf=NMLj9}lUlWfA+RJwa4){SM%j#${xV|P_<#WtS8(WD5j47XLXW^z8UUOeii>V(zuqesf-&PWk^A1 z>=~DG?LxTcXe$q4)ZQck{Uun5|1NN|FM5qUb7RyVh4p((ICCNM8RN-3mHDtVT<%l8 zcheY47sI3oS$scSenFfP*6uwQnFmSl5n(2g-v2?C9wD6tqyx&1yT~J#bWS19fu)@N zK|#2vJ%Rr{`u$3%G~J-=wDbQU-3{Gd*PR#%^8LA*N(=<7)Z^e`QB*qkcqPl9Ut#u!0b;ac8xvfcXYqUj@jfEFeFg3QCFA&8ozyJg;)`bk_H4Rs z5RMSWD%eBWo-l6e(M`v(e{B6IdM^4x;^dz+Xk(<1yjJ2?ZTvoYEJHV^a2bSL4^fm` z&DDYW;Qm0Z(<4dehcE{A&~9Gj=g@SXi+yA!@w$lDrN;R4I{8P^eH~7zv_f=phA^86 zQ*GsL9^JB+Iz7eQ)J}gubphu*!sn(k{}4{)5sKS?Yn%kWW1hpW!Om?xad)HpKV@A% z!~WAld%YJf*YP*e|NhZ25}2SDb#D3wzd##ne|?x7AkP0*VWx`M&@GDFgnyCn?1h}T z-C!Eb_(y8~XY^9T9PqcQ9OR+qQ~w*}yCF>ei?sd*P7$_rs!qbFw`QLzinb|NQxDcHCCpPS6(g83A^X)c=<7I zvx`Z$oag(2cFObMEJPaSX-D>K`LxS4{0ob*CB{S%He7r|C+A3e z_hZ`aD)T($F%YJtn>e(2DRYR4utvoZbQns0BS5vmQ0iD*u9E{Mo&`t<)Ll#94t8-B zo8YX;4o20iLCgE25uW3jNcJm{i%~ENsApSSR zKckbs$4~8_mf;_VJ|DS6`4Mj=`a&?@Wj}ur)V{tHyTz}N^!Gf(=AWVd?f9ergn0f9 z+!J8;dkDK;#XZYiKq=3=!=&^+;Uz=@x0n}FsSj0rl{lLTw}#y4LTjm+$A8^OR0m4+TCSC3bim)Zi3eF1n{hGA$(j%n0Ql!d`Ke8!; zaIBBlcvkJgA1ID@=H zTr+3ubNHWvXf?!&)L7!&(H*MU)i~A8SM+j;=S8%MDT%VcFNxcW-y6u%dExRR;fjuj zO9SqA*|Xf_*#XLLz8Wr@DYJ?7+XxE@m-~?W5AFlh9@xkof*K2jJz>tv@>{R41degaM_PE5Z2-em!Zh96yq-xZz%E;D`gKM|4_oN4B)u7`|KGxXzv7;N z@u2ox|4CVL9}|BJFQa&FX59Rp3ikqb@qCCpr>k}Le)rm7+C!M6>dCWR{k@cQ5+H~B1K?lT<$3np7#aU^Pqs0bkA&fc?Y{5SR zMv=$Yq<0PG@~pT3Tl+d(d`CE2lAn&U?4eNuUk_vK4wGW`6-CLMdl(b%LV=5YJ($b6 z12kdZgntoUh7X@(uAo1vIG+XfF!pQ0q#%vETl{7Ms?mSPAmPZ_@Gbc@mGiEIxK89z z=tTFy3HS^gkVV)k$~zf(fbvKG4PlZE@-I9ai5>-OCejvL_*f$~Lf#j>9HgHstvdOr zn0XpL+^>`UNdE-ZU3LC{5&1fjMhkEk>+cMg0+o@DGigMWtS7w=;uM7PatVDUcncUC zQ@IP693|^;=T3~0f~g|S;!^!L?iJwcKZljG7;2JiUf$T&64tW#V%=5Ac)>`y?kbi!d6x@iEeRPhwJ5n6@ z?QdotLi)f4bI8|;JhWXWZe;0bGTzHs$rQ={KT^u~FeZv4rQ#*l6xyM-jdnbXo!@59 zMEElVzlb;&0{bQ;Hud(?L2ji z`+fX5PqB!WPUH@*`dkcXm9hfcdnk%5x1ek1}cN(p1Lh9s0GA{RwThif1d~ zM>N$c(pv{c=57;tZA8ydZt_?6X1R=!HMj$;z3Y*iArp$&;~ydo?R?hB{+DyXzmj$# z`+y3b8?j;2W7=7zOF2zTg)<-a`zP7}d5ACvzybl*-+jz0X3`5#PX934s-3vxk;nW| zW<|G?#(C1HMAouq5cVSZslGCv1Q-4$?$GhiOAFy1j(%=7$S##8>+W>wTBZwO{@||Z z8@*`zO3Jm5HT4u_I1T4OJHNB^?obJQGe{d>f1*!;1YRZmP2|0W{^R)uY;E6u=9W+AixGP_U1Tg!*1!hBy+ydU@v}3Q zp&*q$tVCadAMER4UShAW=7zH4oa2dqnKX+g2FK$yl|Oqh^^PXcOuae7o7gjy(KZEx zxfe(F5#|B%0`dtm92o@$c!ax}HXDjQ68vr03wsZ>psG6;HU6XVS7{Y64j!RDPv00> z`%Nrxs^02cY#@z-)!eo6{2tIKT0PCm-szkC8{AQ$7hYw(QqSp2HGV79^B6v^!0(XK zY2Rh)c{0z_gZY;&qTdOtY-<+Jb`wssKhiDhkDrMDf&Tc5Rs0*Y-(AKo=~gXe-zoS#ApR~DA3rswpWrUr zPyO3@cS>KW@lg;@8iXlZ%ehPW5iYQmalSoN_UYneH_tL5z#cRQX$I9#CCD8xgM20_ zl>g*lnf%{U&(EPL}gbt#R5sHj%u}CrSx* zGv1~h?s6YvMXmSP%O)i6V%1qa@D~2u`3JcB_g_bT!TvWXQSu(^*`RQiyP=n5q`eZ1q483X z&AEW`PNDtOJHJNNUaaXx)(F*pxd9|?l&jVl#+h+7u-2>o&mj%1{G2axsh7GF)Sgd8 z-$wgl%W6JQ@y&!)bGjN^dFZ=gGx=)oXugi89?Un_i8GA!S)Czc%J$MA^xVh{3{;4jk4`^dnR z1jagJ9=myf?Ty0k3*c3i(MUZ5U-JCx1j)qh??i7)VBf}mfIUJk`4py!94CAMYrY!a zK6L+e+BGgwim0Dje|8eLi1zr5XB*Jf_$^ZDpXW{!S=fUOQ~y8mT#es-DhxJSM4uH+ zWn3};e5sxxRXIQ6dBJqfB!?-_7Rqy;@=*U=9eWJ+EdC9Y#jlrB{ptCKQJz8A>Il+0 z$n!?xH8cNwj{Fp94A$!~>8f`V{tuZaUcr9?{wj}QYTkgiAcT6Y7GYaXd1$xKb#j(` z+_;aa_)igk5H|V+<)^<+sBp^vS-m)r>JGS&dKqIlHwLI5W5OTDo@+K~u{SPcf45bI zV@~)jvihY^nSE3B->4+>p_+>d ziKotKC6u=?|9M%4G;%Lc_?YhlR5;e6B5bAg7e;{t)H@ z%2dMhU*IkoLphlz)a%{7oZX8^^F!Wa?|+W*hTmA)#Q(CM_8|TK_Ty;B{Z_ENwMtk1 zq*sq_zR!0FUmB#A_4FL>i`4mx;p`RQEbN<2d(PxrflYe(Bkqf&y^nf-iC#XDc?Ns< z=?LDLM2m6WXlxa!(wL6!`pO{9oSjt~7hluMEBL9n7hfd}@}GkH%69HGID4yaEsKy> zjC#q&-*lLF)wlHW$rs#PzpkCJr9_3#*Q@&vDMSC{TfJQQhI<>oLCVi?r-+~Pib1M~ zQ%xD(XP@~?{Jh8bW(Qe;+=R@4--5Cqe`lh2*|3{%cfK`DJm?+Y3={XO!=w#agnvD> zwhfa;q>IbMqlXRRJUmP)a5ueZkOs!R0~>D(<$Hl82Kg9&E7ZJ1yV3?{(5sQ3(1z9` z#>#Bw%6NmkPdQ!4{cAbvCvoP-#y(2p4gh%&4i)I+W6pNwgL?7WMLw`H{yp^n3*5Wy znIrzTxvXz<YbZ$I%{dB**mJcr-&pg!VleF1x;Jzu~*7o0qg>z6B#2WhiPScv{JQ18%RL#~H^ zf^BeUDWBFNbKrF-guQSYj>8rBSJ(%6?1fAZb@B)FT4;n;puDQjjK=2Rxak>0`%Y^!Tr$~?qhKKpbJibv5~c>i1iUYk98{lRp!dqL*z8>eBOr}yID`Cvp-=T z@-ntA;C~6s*izxa5OK2hUdG)EUh3-|7b-3630xQXo(DfSYjDS1gM9SbIbHr0?hVcqc`2`wDf>*sy%>&I_3{Vq z!z~NK-|{8Y?#4aH+pPQtbAT(x7`v91o+VW@ebw zLS0gr*cOFJJ<`sZyq%=Nw8VtTM&xF2;op%K zCT_}=fjbktwWJH3-~kJ1@4$Z-m|H>2Bh)NMNNAka&|ahrD=V266BA&=VW zI;q2sZ1Fm&N7`{WARX9s8F4D$02~4fFhW};_vv=VmlfM$Ztcusd}U)RNFVM- z=82}q+}R()9--NVZF;a#WGnHiNzVoja55h_kuGElw4xU>w#@7+J1CobHuw41um`-^ z?60%9kGzdP;e5~qC!h*HE1W{N?WAtoc&CMa4jl8S8*_s_nYzLS{4T*|=!I+0Q_tL{ z+;4-O<0~^$XeJS?74SR?D2{AH@mE!J@j2Cw8n=?3%HM#7AxMASn(h`kByg_yW^!6>c9r|vq_(K z`VC0O^zm|zI2Yg&T!vn_1~;J(+OEb*`|ViiK)N52<_*#=B@giJjg`h^z7JSCUXCK2 zk7K2oa4y2PAX~fDcL#i%KspcL5j=q^PpnwMz;V*HpShE?3X53NUuI6A-VGBeqiPEi zhv=ou1(3U%@wI=vgcHYAJYHH>@C^WZ6pVz9>DcLH`W!pQC9~D5DKo zkK0~}jk1q$m{^ao#c1M>g;aeHv9G3H}@H(Hs0?94w% zr-%6m-9`E><&3?x%pd1tWESbnf%%XKRWa;&=>LW2wluxerRZfb`Z92=W*>x~9qPB~ zWjTH;VHK=_b+8^b!e;pI_Md@!2kZhfd-uZELnQy@5Xr%v2l*kP(mR3q4!x%an?d&N zds<46Wl#YJ;1F1#8k|R%H;@jcLo2jov;P1$x_dSIj}->-p?86gdBO`G=wz;`S)rF& z>QqO4R9)*`)XhU3kq$d&@tSZsLA+CN8q~Mr=a5FEdSB6oJ-0*0VEPxnpP$iv(0HDA zyxn|1g>;ZNXhCmH;hTgu_%c>{=fbeDa)Goj!DYAxH=z&i!2|Hd4VFjBFMq5k zEJyV61b5-}5HWK&GbGY~+4LWTquW&7sN+!dC~!PxY=Ir>7f>eF$zI}*s9$ZP7RiN45aD%KtUkB@9BeacRE*wJ{ znLFLkd7pg&cNkuz?*{viB<9j&=G3*!t;lBFu9?ia&ubQ^HnP3#|#X8fEz>@Scd&~i0Q%8(V{x=Xn{l#8}Jfcp?=>o&I8 zSxy@=|JTG&|H;8UiVay$z3tSaf$};itA#k#U;_s@A-A19c@y{U$hO;zzXHbJUdA7? zvl9D9dU5-ZjilEULH%i0=T*iZ{q91xK7-rC`VS7cNt`~o2M?g0{re;26EM6;|L-@j|H8(t zRwXP-_c;qD#BdKpbnT~q(RV3-Y&!?p!B}u7F+P$RA8Q#ONZ(TWe?g?o;&~pJVK-FaXHAZj z5_B8y?CW^%UWQ%)4&L84?2n|6BBdUwzBO0xrE+69m-4)C^k6xJv_LiV#zlw?+4I^k zaUcu1D>8S|7wD}}LprsjQ+F~_Y+N4K^FH2wj6A?`KT`5}Z{a4K54zw4oPyJE4%)bA zZHEqULnnB^3%>hN(#UrQO^YJsXi}s&k7p}og=!1LEM)}(*e}_Gia|-QC+j@{*Q2X=7cmjq6*iZyvAROI2n6!0~G88=ueA^<$i`#>L=ZHwjrTtYOYVSc)`Sy*z ziYDAxu%g#3qFzaScdN$|%%a64vG&t&R}f6ZF#FNHChiv2N8>(d#VO^i`+KtA`g zv7|K#roc3q0kdEZv~9!wK(ljX=Xva}8~a200@&YK);~B3PH0|%{Xq+~!hF(j-q1@T z(rXOieoik7(HDa@e~i$J8Y;5A4qYxX+idZ^mT`zPdP zuu;!CuqRPRa8Skus5wvlyLmR9^WS9F#CX<5WCLyoGK098umg5M4wzv#lt5bn^KUWp zZzc9sPX9O2|KQ!p`2!l)vj4iu{;P-m*F)yt8_d7m*ynlXUk_{l9c+tqdY|HZ805gY z@&J0Sox6QuUB4A7hj3e<8f?(V$7~Ly6I!9*^=xq?ebBps-xDHxnsssl*>^ZnP9eLP zGxNV?&PG267r@DylZU(nm!TJ0@N=>DUPE^uVGUz_y@}ojKF$nYJ8Ktoy7;c1@z?h} z>v}Tt0OQa5xc7oqp15E+W>*~9tW8YWTbBOw~bLKS{i z(vC&9u@_2MRe^5`0Crs9m&N|4?=brREeh(Bu{YP;R zf}agp2Ze)ZPu}-BvMJj(%D9uV>cV6*VVrixDKZmUYFPu2yTFBiM>^v&i}(Au^T4~7 zd1*G|G?j6h%>E;#-~N#{s0VvG_FsvOlyin9&Tc4yGN^z9a0o2WMwoWOb{xS@p>rGd z4_@#=<8AE!KK2jJx$f_Qmqr1Teb(1MyJms5C z`C=&FD9V?umoA>2fK!nF;@iBmWE{rOKcT!I#m{@|ZMlFfT$0V59D5V=%g_tgpr(@g zAIJWWkv`O;+XJM3m-OK#Vfx@6Jb;sG|ABl0hK1Ds81@ex;O?ef1GMXX?Ef+CTuJ{g z!2aPVIH4I_&;qT{GdE1aNvD_Z42B}3U?fCC{=RJS*1s)d(PLo}c-wMh3bJoF->D;K zz$};p^Pz|FlZfnniQkAJ)!nTT`i!*6JieL1Z#j&_9#_X;19+;tJZ#L(LY}t+OUi6N1KnZ#o_|U!J zNq&dD$UFR2C0i=+KLCfo0#!-bVufmSTfsY02R3wv0=)t3bUhVids{Lf?j zLlkjFLNts8?H&)Qo#RGu{vCn+kHP*YV*k*2o^`pKc?{{pjvAM$Z=A9J)z~&PCt?3H zv43a%phF;7kqc* z59NGgkXg80$d=m%nS(wb5}_lV^Fb>6hHU01q!-jT!3%k|7<{a=mdWfh(5=X->D+Cg zuY^@lXQv!i<_8b?A{#)>k860g4%Wj)*bKdXgJdApJ!&Shk8{%wy`b5U?aFIlpp7f1IXUjbiB}IK7-uJ*zniy$RYeK(0An>sYW_=>|2n9*Ex$m7a|Vy z7Up+XDDw@vx)bf_;jUBZ;A7qLp5^|7G&}L1gkPERhb}mx3z5SkXp1rYxw+#ZQp0*x z%NkX8gZ1$-YtC)fp7X3h-K>wSpN*`eP08WhuW=We&i#HC_xq!`-;d#bAK5mY_XD$8 zhi9@5N3iaWVciAqVAf$!zx6*szNg?coP(+?*1ttzasl1e#QFP3xRjhsmrJ-EcR7ET za{k`W`5XW8?kw4NY`athw#o<7Go^IxCV4M;layWDF7GF$N#)hIXk++ zN6^Ff5C?a1-^}0TI8uPU;%BLikU#MKw^cWY-|h1ImfFEmDT-8GrFsszDWz` z-i&<|)ItL|pdQrkD@Kw>G>nBTb#Sm-KDGr z`M{)j*;rxqq1~~to=WN@}`4`!W+man2r%C4=T!2eZ zMS51ajBdNg8TKsq05y8)#qAhF**N#wk@d$o_a@O^oIP^Exgtug5ynNnEpQXP@Cy5w z5YDG-xgVHL`DRnTd6W;`$L}&~k~#mcC698>$#&9ok+z3CkPh5^q;U@(z$179hM#hd zfuZpI{V(?HhR!tF7`*HSedpQ#ou~d*>Q7qEBh(+=h2Bz0{b_$oOr%7S&PXtjb~Lgo zp8CUBbXy8#qrS1|lfaQi+3>SxQP!Q2G6lbBFau`69GDM@un@HK$|%~0J@I1P%V0U= z^L>JvA9|mPl2y2Su0&u5+{rgGR-fXY6~A?`9yY>e$bi02qa+i#19m|Uh8TIi{A&Lmq7)nJ9syAGX6b`fA0;}-yXhA)QQZ5_`Ty-0v;meqky3V_U7zHj(=Y=m0l#f(N|d10#EM^(~c4Pzk| zoQ$(c$SE)lWQRsx4P}cXD;2G04J$R6Ua2$5kNc&Fa{Ck}9Z-9GlWM?<$U!)he581ey^Di7F zEhjY35urT#~FXF&Qk(yu!iCWp|yn>hbd#)g@cQI++0m{jBMyvw&n zn}YWPEo<2i;^zQYGJU_4c?q1jTfrO8yfl%%pT`)O%`^Obkk1_HHI9=m^d9cHPau1R zIt==;oI*bhYP~s!ya1QrGW5bVu&}1yL|QNEr4M-zs?PFvH;|9Ob{rc9dpow_!baFn zt2@Tp^iZi;&DtNqbDpWQy*h(9LV1p#cTMBgMyfr4np@Ny)0D&+1e_B&gG^`bpUfU$9&7&y_5kQL*lR82tOMIs@_tMiMsfca zLt97i_q@m>8pc8_OoAye4Q4%=>R0@2HXWgj3)Ann0Zkb9kNyW(Du+3g2cuKrewZsDJ}- z2rN(yHP%?EweyZ0Y2%%HJ<^W50qGcpA7LEegjR5a54zw4v_f>I`+xqPfRk?wPLa-OI0qM?1wR*DLU+6P zegQ6{_kxdi``&`_;wc?3omT#Cz*O$v(`X0QS7pOh=+_8$6RhY1-!=5%&cDLA;+qFY zfVS(Q4bSua1AS0?m2Y%zW4Fk9`pS-M!0kYK57S4C|9ixL0ENS`^X>c{2lOYAHTy?3cUG*&YH7IFW?m~$eV*D^McEx22ehNajQ4238d34MHn*>`KaM5FiI ziILtbtY6QMm!5o`jKwb&CczYNVnh456Pt!U17?A;D>eS+pu4fJ4#wzw^hEIC_Fm+U z0DJ31F2rv!ECb68#xm0Skn=bCO7vB*2I|gZpUemKqk?mUcK?Ulm_oaQ8h`5uw-NFs zguM-Wa`eJaL&W?F=W+B**a5pB2lAlCj{SSEf0s(1_8&$2$I$*rW&i5^hnaA@p#;jH z0uI0-Xd`Un)T_A1X04KD98+_n>BSgB8rYzP8=(`fDo#{^CcM7!q$J^ZPqpLf>3&`HX{H6qX zS?SQ%%=fFvn@Wd!;0op2JLEZNL4SaJ1Ws$HJVAQKu%@wg`BGTZkcOWzUcfz=HE$y8 z*W*wbiaQEMLNrvZW&MJ&=(ZKCUpu+WN1p`R`n8nxYXR#QvWDlis|iE;HmC#`#qB|=c3Dgqi#=SH zA(umym2b$9tH6qX-6HCh#2r8GbRy{l{nH7D{J-n3zW-TIyp6CK zG9VLnz%FPjW&E)gcCfE=L+3W;cJP7^8t*dx;OKqEALFw5IODI0IzlTruZBtv>EwYK zc0)@~DD@AO5_C6q+yQ0i72wOles$RIVC;9IUJl@AIl}uXC83IOQ;oEN)y}zo8+HW_ z+}>x|w=Y7Ui5;h6$4S`ldF)r6cYC;_B8>X}$Bk6q|M-w_;^7+FYmyj(U&HyI9OGbBdF6e+>^lRWl_kssHp=JU0PdVx+m-<$wektY3 zX752g9qnrW5g|8;*9Z3?f6Fj=fP4f`pbdL&$F4g-ee2hWy?aK{{v#+qG@j)C6OJ<1 zIiVTd1ucYY&BFfQ;_Uw+=P>@hP!E4!z_1Maf}s!vBOw~bLM+^({(bx=qwhMlfm^M; zY7Vp%@XbEbTCA68_|JeU7w3`}`R?HJVPd0xs*ZN32M4IWsM*W;x3vHKTg#b}^QoQl zX#>*1*?SV{%z`;E9};09EQV#!mc{$`Y~H^k-TXa)&f}b~`H_3huipER2|OCJSi z0q0-Ny)Mq_Ezk;1&cDk^XCe7pm>3#{mMQ#seqj+7kSdC)+9j0|z*v z>LK6%-er6#e{8-E+~_`Vq)@k=)NvbiMRwuW_opa1fjkAL;T&9mOK=%_;Tqh8KIOh1 zDff^)>Ia0Jo4-FlzsPwHVz7T$hwdJX{mH6F&oXfS2FA?LxNEx9!l8 zMjJyXZR!Cpx{vvwk-bIJUhKaB`%kC;dG`JHe}pNVK>G}04koQUFvD&r0cGc^{yner zjR5Xm+PeaI01kmgk<2?(qzxS41nv9H7|uA#e=v2M$N31|2j74H_mKV{f&Hhk{(xGe zZBUPHhlWblsq!$%Bu*EcfKzZ9&cOw^1Z|vS+aIzQ*vt91fHjeL9;6q1F3!KVIsX#o z=wr?*$Y$KGvz&jS6`XfD|FUm3xtPb`GHJHlAWRQw@Y^`|OwyjpH;VXQ1K%k8aeF3` z7xE^47V;_Np2iGScD`B1?;cpK{p)HL`x@Mjpdp=lt)QMrdn)ViZ0bLca3<;x^$Mx-+d zroc34(NTZ=XP~=BQGb|)J_mgCk#`&Q52gMvA3sYH-;f~k&jT~;h7u@vAzsRm70}nnoi*|hSfCng-~jLEapFYgR$_CY?&fMD zu+LP+a5natg?+BZKB0j-KnFCcGd>*U%;N_6*C z(%zpStI*eg?>1?J2RdOLewLmDS&!TZRp^_M8DPCjy}GFve>)-*_YP>NrCv}Eb|?v* zEKcgVi)T5|GBrW+kY-Tj?8v5GixOlv?h^1)E}w_Ish0AqERC(|2v(juGBd+ez zQbyYOgE+7AUtu)qMrrTzoSrek{f8afwDOyN!dTej{UIh^j_||M!`tHIljD4=yC6iW zbc3Y2XRxfKzmDAJ_l!5VH|XYl8o&9nA0HtN7kOtmZ@AQ?4w72kFsZx9`<}rO^6tzZ z%kCqyc^5l=0{Rc z`Xl)$B|#3*4~1W_j{S+hAxs)QuP4aCVr)SY`65f;K?s9~+CRqM!J){z3%4c0q5a^UxYjgML17jV7?Cp3d=Dd%g>-mTC! zk@GopOy+z(kMsFd&gY{zpT}@Me@vXotWD&z99F_A82J8h4Z552wEF&V9r}9kai;d- z=W&r1Y{bu!!gp}U45&iSMD749Wv^RHxqJAQA9oHkpgTC**Wcy+G}JUvKM&yogzKh$ zNcC=^;Xdm>=Qtz#BolkU+~e%&HgKNeZz`IF->8t*ZfHBp{_i6DzaI8~&{@j_`Vxv6pcoTfvIIcCUD!vf=iDcCWaNz29xl0Gva5p5+XHe=on; zK7l+1r{Nr2fJ<;0df^(VZ{O8#Q2Y2@eC|$>oA~uX{uk_T8Q%}kUC0&)?*BP|@`tB8 zYFU4+jC~I_aD->zD`zZ@pq=So^(~`amDfsL@T*xgPinW#lSicY1PrfWComMEU?fC? z_cr4wiLr2&F>#f#!X1zM7~^g>`?B+lL)xP?f_)d`%!Ph*ChZf?zH|lS7HZfV*Fv3% z{RPy6`mX5v@1M6s$XJ!n8xazVoCH%~8q9$2&;M&_<7E0IiN2wqwcmLl8;`L6x3mAZ z(mw5+|7$t_S91O@XMfIFpoia7_I$}50N(-jvNxJVnsZ=2BtkxaS8*Y7F)V}SuoC+C z+l75!^Zm?Hy{y8$2G+qwsKU<*o6&7ksoP}6?GpNdzr*hsL*1rRN2uo>M*IHV%K6vM z`4?I5;arSt!0kX9ukv>WZ}WEok?Tn>12SO;?1HxQod3Hy|0CTuIR8K7{10AqA2fDz z{$Ie~I9$Y8e{GaBBV9?H|BzvHloOJTQ47;HPKi6!2e}L{TAnh@nf6yzyS4`R_ z@<4W$M#%yEEGe9sm4qtvYNQRU+o;!C>UAti9Jrm(a2!3r`JeLIku_&I|6e5BJnrA2 zekyhW4cM^y{ckIA+~9*Q(Cz~`r>NfmpTONln0CVc_5O|fcWlH9K4`?wn(p`CzcGh2 zcVqvqzuv!{B8}7FY-h}T%GkiB&*8oR>NmgxzyEbpwvLI6tuc%-%IDij_z|q}%psQu zdk-7AjO>M~8=U`{i>{&DuyysfLT{q?fnz3hTTLBTP}dax1~h&T;1N6lb?;_aK{}k} z!;!vcN6AoRANL(m$dM2YV<8qMLGO0{{|n9!JumP*0`6%r17^YZ_a`HGw}pE?^nDT~ ziO9uJGp7Idzl(Uk&)&SAwaU&~+K|QmzLLLx^vNjkzRkI05c`X9{0^Ns%V8y~g0}nW zZyfRd1L@w&nchVEgBN_zc$MG(-sbneNGE6hW^~tCe*b%(`^Er&1Ccb=!Fq5W%SXa)y+kIK6H0~+jOCy0N)YfF?K!Ov=!8_*3`O~cRkFz$a*{TFza&z>$3yt zB+e-~4d>tjT!PEc3vI0F?a*yCah~R^D=va zRx*rl7$F+>|3}*021j+BiN0S7(qMvRg%CmrFA7CGgux!dAsmK-vEm{Wp$J7N!X;e7 zC0xQ~d@+|Wgkd-s3yhHBAq*jy5P}&@Iuepvt(IuD-dgGoMO+L;Tnv|?7>ZCtrEmy^ zeHDsOgd&*x+l?X4%bfXeKK$!>*4n#Q@3r?{Ydz0;+rv?ciZ``8Kjtsd^Xn6SA^&g5 z|9SaG{Hpxlm;b1YASx?kojV>AFbPvI4P}^tcICN48SlKR{@J9_Z1Ul1?n4FGFsscWuMs#1__AX+xa5*Y|syzJ?cwhe-yAk)D_Vh z;GDW3rG32Rf9~KO9-wu!dRTnLmq&(-@*S;h=)b@_hMcmU71xcfX8VZE^`4>c4L;gZ z>vewZe*D8ow$I2=;{3rFis2|lG&VPe>~$~4lH)M}lQ0F-P=*=kW9L`AWBmV{-Vx#6 zFYEIp=V1XBAsRDXLN3D!tU|PQqFVfF`lI#RYw6kA5#f*5Z`V$={v7p)qhYCW>6ym( z=NhLb`}y6gZ`;%Vc4^ppzcj2D-iZACAA1nBzcng!7|-q`Gnb8tlik8uGPl_J`{~x- z7g>K_Z2djiGSb+(`2(%=w)^5bMupANE5~+Jq1A8G#>RKjGseq1?~V$)>3flLEPK&^ zob_MiesMAB93&5;W}ER;@+hkJ$XB;~?SOC+jfdpRISpiDpYiu`@{jt3&RHXVgZM4- zPbQ7ur~IZ38RT#RCvh5Q(0*P1Z@b3(@=tcX>)QNp7CAJHl>e*pFFr-KI4@1M3b&D| z%l0^s&IMe;6+GHMq2jCRl+n_b&+KQ6-8z;N&QA3o6a5$Y=>3cBGZ(@8SDhXedYyLz zHT1pS=RSH|Ug}*>Vv{_sdziOIGT)D*e?GNMIdEQ@yhGkYEB)KXn;y_J-l@)az4uqW z_jmPOomCF#{l239MTIo7_W$V777i5d4dxL1t~8X;d!8E=`hIQ>!PiT}VBw({j#AVW zu|buiI9ZQGF}svZPG+Z~;-#W6#(85g9uqJLQ!ou3*S&*1%JyAloa`3vB6I43EZIEH z{GX@U-qYFOqm^s+ZwuSE?IIs#kadUBn1NY{?OZKdMs{XmJ{*i7Y zlIQvBv$Zt~*@K9w6S6-j)E~nxz%sw9wC`RO^6LxL-3^1-0Y%!EVs$dP!a1w38hth(1C(lhCY zw&{NQK}0@jcANHtjMj=T5O)|c97P;Sq|k=i0`t02zrj3jzITJRIH65$tYaUbNn78{ z2Rn9G`+i&den9(<*4MP}+xTH*2RhfV&sMTIwy-_OtZfp_*QQmtFBTTSoXE8aw#AXgDPwZ}C6bkaeivWk2yl_Jbo6-PY-*?Spz* z+4Gxz3zabpLf1}5x*WAu^Bao^lynTr*B6U zc49a7;)C|fe&K^Sj0$aT<-fZA&+4ZXjw7p|KS`$0h931@hV1n%f*e_4{`?8D?_Yfn zkvxqvIEM>}#`G?cSI~=VXjR4|`}+nxGhO+cToC%`w~)K9{2`hn(1knVdVXhLC9>DI z1s;(7{x{DX15tvkIR`_@;V8uzj78tajNdF(eh|B=k4l+}_I(=f*aXxZ;Ahl687lQ5 z#MJ}!-Rho0>L4UHsDn_;x2{7ypF0{SZ+MpfKcD}PBx0^-u(T#&3Z|h9GcXJ7@0$OA z`~CU2Gd6$7yk#;c+%%E@Kh^kuf$@K|+~wP&Rlh*ndE@V!{6Ferpn|!Jb@+REvoG$WJWi7w{ zURjF1zf&{QaqaeY=U2Ob^>f|Vh1TC2n~dfUH3}!)lLj*3+$!ho#BS`xejLOk_x&(g z>p$vHKTJAkpeL4lK8X4!9``SJr&7217oLCGbA2?wyo z{Wdc4=c7ISqWJ+C;T%rjBu?WDQa6jkIr0K7;R;%}6^C?BapOZ(c-b2j`#i4q1ad<$F?_n3b#?DxzOdzSvZWKhj$dCU^>ks<> zgd2CL7e*F@Qs<1pSd7O6Ou`gYv=@bT_qt#?VFqSl4(6fnb9{=QJQbp`{~nv*_L^kb`^HmYfn%>m?dBHc z9*cyRAp2A8?@a4;>1nccuD%R<<>%%1yVfvyFEZ+guKTW!%=Wmx^ZJ$dJrVx7=!tOX za&dTL`jg?{#3#cKZx@FH?|vlwVERYG{_}&v_nU`=eMKJ&uU{P!_NIr0?@jzf*ki-l z?-q{;yO)m$w>+m5|9N%v-`d6%1Uz;X7}CBq+BV1Hunqu>KE~|MDQ1+mZh^?$%BAU>#i; zUR7Tlc}x4@cd>%sg&$qy&kwmDew@A^j@q=drr@{A{%=F=zPs$>+xlp4heY#DedWLQ zZvI*y?ak2K^KNJ=ct_p(t8nc8JKnA9`WUW>Z`lr!KZ$oTkVaWdMcVUdQ#$r5b78i!<$%SD8J-*bnVG?}` zl83CZK>`h!CN8Bvy^NfNR`tysavsth59^-H!l?iIeyIA%??RSOk>d+=DI?uv?Pd9& z%?2@E7GJ~W(C?Vo#|B|zBsF$u(cw)$-UT*zOUM!hdhi}6#f@$N66Z__w}#b z5A|p7hd3H;-Zw6GKQtnV8P18;Mr|9WzB&6K#D$Zn_?A4PeZza|zxULC?|J_3dH(Nt z{_pYs-wV0h?-|d&&py8&jvZrJ@ZrK-!-(lhBoJ=7C#967PBXn$NuC%v>tJe-6BxL2j}4Mp^47yR_|fBbC7+{QsvPj#EW*dm5CR zr1q;(Spc^W#^^fh9e6;>APS5)% z2b58Iv}UnheUPHJA>aQdU-^9Qaoyr_IDwNmjWalhK6Yfqn}x;*AA}3Sy%pLD@=B!R z8dBPf8|03k4GVqbEkt9>>AQRjdZwCf;+%W*2gn`r9-v!%SAzX-%(bqVpE~H_xNF12 zsQ;UPJx=@kw7u;7=G9Zy{~hliwi++*_rH5jk?;TeoqPW-|N6GJUAvzlyM(*vKWy8h zaoMJ&{PWTL^QruE>9!2xTchdZ0kr~8~DBlV1eYNtfT}~?d z(HLPq_6gQM`1-YNCxP@V;oa~0^Vr~Jodz6(90?023u z=0G2dT(@W6>>cRvE*!T9h5s8begY<83Z|h9Gf+FAgbiB4|Ky*efu2BP9se9n7x~xk z@~`jmrEl}E_wld!?yYD`mxOkHeaCX|A2R4dH?qhfwzT;7>t9D&qwYUzUbFsPAHp1Z z`~cr~k2M1H1xViK``+XO-{lK$dnzmv7yWPLuk2IYrA~J(OT;Zh`l^0`O?=n6<}L`Y zLUyu#fv5Eo%+^mZvxNW2{};a&>#-3v**2BvQzahoASnFYA93u6;rO-+X;Dh_BQ)vqc}x9_=)e$T%m5eEY2p zuhd>uRhWAq?ljt+(}B+DTy!!s}94R5WU z+VFGq)E)K#!w8$f7t)&z0cu4>;Ej3#u$vn zc=UZCgb8Hj5A@srQa^&VInmw|J-@ZyO#IfV>?}lUwx_1gnEwAY`v0|?4cg5F*@z_nz3Ex~|8MF4XCM5@`a1v7?pO!=q4T2o0eG};fA)|* zfPH@Bc-;RbkwP1ywR!3D&ZlQa`>o&S=qHdHr%n{tO?FMTH?z2yHvcqv1~v3^LmUlAyr%wdc3;u!H`j0jeYk}?xQ7R5XLon# zckLuI`eVAtZs9DMQ^uQ)sQ(k{|7!Jrhx*^M`u+afzK6J!eWl)u=6r5Nf8l{BLDY{} z(eR-#m|pdSvi}`>fzwYY6T`_;q-Lwz$*~xZ31}6U#w2=XocHf(?=F2BqB$#B?_)Qj zwd&b_?-$C%_x+ZwL(amSD9i?!M@DVU0Z|4p82PdHXZtO)$eYKxV zU)F!d9ymxpj2JrQC0fherTy-n>b|+JIn?a3{>Je*s&}w!qVxGgDY6ZX>%2#3SmV7~ z9+gja1~QK2Z~{FQ_FEu(KW%PA6n4+gL}5Ng_E#grIr;_k=|isgow)_(7+i|NU+f>Q zMCaieZeZK{?@=E;S`%IQdGir|IwIT=zJ~|s_kXBQF%Y%Ji0g{1zqgJ)PBs)+e?P$b z`>EF7+jE6e{b*ouDkZ%f7kx|2kgI(rl+mHf7beY zX{D^UZ=t9CSF8VOGoK)}qcn_{&IC-t6tuqP989A}X?FIMhBEpL!Z`l9tEc@@GGST%{ukb6-x z+WZLeAmXmE{VqCJ47(DUPuPupjhUWOc-C5uPnkF}HeAluj}_;d4r zx8DE#{O?(QU#Ty>%05AJ=oQ~A4DGW$e{?SP{Lw}4Ms}0>0r~mAb><#)yVkV12mGEi zS{;k#|IU-n0xZH3EW-+{Lf^lR%CCRu`I>h`xYykCwd8ti#AZbEf6K}3sKQP>n*Y0- z9)DKe^sVfr??-Y7zYB@c{IGHQUiiLseBb(o{NI)Q-z|JgGQUo4xi-OX4&pFk$lI^% z#%TZBqr!Rrmk-=|_kI7L-`joY1O9Ih{};!Qy3haR^QQT}(LVWY7sXxnztTw}g*HUT z(zm5Y&pa({RIzC54Ug!?YsE@@*ItAWU5_Lw`*JH z`6Ivn-hF9)TmGZGqUDslq1Ab9X>kR{ex>sO{l4zqN9!=>4C3?A^ZO6Dz7qOikG=r8R&b)dB1!*%%VqoaVO?K?CVRe^gb3Sj|053i~Zk1HpM%| zVUF|WVF9wgEe?ywC0K_1cLdlA+2#8D(EhIecz(sB{l=nip{CFC3+1nkY~nBH`{RXE z!oAwm71D{m!?B92_=-LC$+cLIBiBC^-W;Nz#(aYx$**s?hK=Gkqa52&g&m*!%Mk5F zu#=uX;Q#2m>6P|NtMb9c&K*%7CHq01x|izLBy+<1opTU}5yMf$vHS9k5bghy{Y6RW z@xA(_xZbUv7dvMg+vQcH#2r{u8h-FrX*hUlc=+L_PliJqJ{jJ4Yj`+(*M67YqdhNv zJbbtLM%cS%X!zdSL&LtagTw1De$*b79}VAsdvMsk@Rz>Z{>!jeT5YJ>H8NE0`9#<; z^%M5M92RyC8sQs#BSOZp9Cq&*68`Ccv9GBg3E$Z_GVGoIvGBc_9}6cOJBj_v?SZ-B ziLmd`fbja=A;!LpCnk!+X~)js94_Dz-njW>II^ZVT%pGb3d4_fcxMk3h9B=K3`d9k zjc*W{`!nEgLhZP}Q8ul^DzNUV;IpA&IQ=|_=AIk*#B;@qHWDE3N4AF;yDdJ*#O2Ra< z3^nPJFoT?h>TU9MNWNY$k3x7J8fVIvJT}aiPtnmpkxgrqH}Cnenc6??P>XZZBej1+ z^bHmpqn6%kqza0|S~5*;BrDW8t=1=IpD7HRg)`dcXdZ6&UH`#;&d(#bsqJ%YJE{u+b;@3)sKxKUR9sGrOmsnzHx2&@B6skTGv!JS$*TWe*gUu$F zALU+Y?8iYI#?OL2EAlAfXm>3g3D;IF|7`rO4cdRURhG;NH@U`U*L>`_{G+8={{4Te zd(&3uKcvxXpT88@^BeOn$P98gfr?KRg_GoIoWVKtz2mzS^0a+b7z{`WLlhXwaVJ?USe-*N4z__5JyWWb_??mh1Ws z(7M?CL;Vx&{QM4dB7W`XOhXxJ^gmRewf_%2KEQwb|5@}o zNNRH%kwAks`h)wQ&2v2a13o(z&=;XHSWo{`^Y`h?umY>F8f#Ijy{=p1Kji1n^*^k4 z&PHrTIkuw;JFy$>e2NafMSlO8Q~X~(?H}xa*3AFt;D5Zv|3C}7B#l<@XPfskWsH8W zboS#Q4x?53n?{WO=>0iLk0UD6SwwAE7qVY>UY)&q9ZR7`S#KjVsHWFXRIa>pIpK<5 zup5+zXkYz?Vag%0-+nHfbnMahJfk(K3HEy9v+N5p-`?FtpAJl|9L#0>8!o_oj(r?kp1ywOEhN0{#iQ(2Xpj`AM}qo)2}_wAUm4*7NGe=R<;QJpO!0-hV!9^xMrS$97a< zCw5~m+UJ*P|H`y~W!k?o^6M?? zZWpo#g)>WC-(1%>-Sv%geWS}l?Ogd^=zkDjDcvp7eNFy5{0~ynXhQ}$oWMz(#u>CP zSN_n647$*bEOKaiTmH|>Kcc-3TI@xTu9JWFq75&^ZK=XykY*6y@Pi;XE*j@KMvwB zVmOL8+U-fufzAW=FxX@7f)0BaAbZvx252hs{uFzE$ka&h51AHjCEJ`=d&v8n@Qze_ zf4jXCWJ0)+Om4A%Km(fqN&l5X8#2h@1ZuVCb*LYw{XsOQ9PKTeAO9EL?wpf2jWalh z3%G|~G@CO| z&#)yraf?0}xnX=F*OmAGU1wD8V^`IiFR)2b<3H{>?*XcBYgaGI7xLe~ddaw&wl=v` zKDWpxy;d6e{RdzCfPJw^{)Ln1w^CWcK$KuGhGIBM@jw6mrt&|G|Bqwa&6VQ6sIXCV3L(g26e@vlILyoVXMfXDfKgz^K-%y%C&O)?z@*Hv=qHicg zb0^|hAiM~T1@eW+c1a-r{TuC2e*fD)Q~sCu%`!xLAFLu*V=dNWBmSWL^KrWlDVxe= z7P<6)UjDbP(f?4b{3O)hY|_orOgF0^P)@J-#fQRnvQxU5ebV10{fW{?4pG^!a(u@( zUkp3R-Ke>590Ggk@r&w!v&IUd_3vHE{uXr`qBX`1$E{~{&OsbT41G=dCVpy84?T|F zm%X2_>)U9z4(I3kO~s|qh758zfr>9^*S}gE^8I6z}aYs|G&=se`%$f&HrD=w?(u+e;cZet!KZju9sE~ ze|ZKu2eE`Pe=_>YOTsdG zW`k?P3i>MK_`g|nJEv@l|hjYwj(->k)YY{X`iV>_zQ?ml(w;{TJG zL;Qbwca*0z|6l)Z(@puuF{Iv-|KrvcpcQS1-F+(Tl+JGK#eUQbGXG%MQ{f;zT9Z+a z!}J)EMXn7A=R|uL=HIs=p2tZ2vC;X_IjDG@4T8$g`!38cjdh93Acqq;iPNZkjsLqx z`QOJ*?_igoXP5Jn8$?8P;-D@Ps5Q77`>-ypkBDW5l$Pc-e~ z|IXC@&(+qGElaikWUFu+8O>k0A)P+l!X32EWIwYD?$I;bq&?d=Zt4A2xqjj70spb* zp{>zhTx_v5hhzzA3XGSMLs6}6s4tHE{(fP&a4Dj1{w0MQ$OPM9tA3zu?;00ZUi#YY zU*Mb`{#wNkjA{IveT{{)?-Yg!WZx4ZOv)eE??U!|!@7=Op1|Llhb)}xX3LQ21pmI8 z%}<|&Ip}B-djj zHX}As9|&3TY4d=|D%6a#{&)0;!cO|5b>6$_`;lMwbILf-apOQ_?RD)xqHny$(Lhfi znqwC2)jP&-UVZ!FT!CoLz+T~lsQkpk`2p=`^?zT~Mrp?~=-Q)gI;4%-r*Hl>^WSD` z|L_Oj-@U2*LmQfU`0qG2fd7t`Vf=Tr(%X>VU!a)(E-r&Eaou>dzwcq!lJaiE$fJlO z8O5OuwKvUQ;Lq3d^W&F|`=9#2`o$yubDZmTUJfU460J+MzoU&K&@dHD&o=+SJJG#cX)iObeN zB6*Xoc3(c884)HqcM7JV3^OnbbI`FxA0(o^w>uH-z1@WzJ^I#J^FH^$JxR@V59sNo z?g83_TQN@>3$O@Funa4(3ajx!|7or8Mr=maZ{4$(N3shFL-t=r_8kkw@0uS#rtX^`K&G#oA27`P0JM)YzhJU-rb~zH8sv8)%>j@` zPFf|>9E^Vcb0|3+r5J;;s6FfZ0?5w;=>DVc3mhs9vALtdc;`*PBuqh#xN1zJ$K}2L zg;Al5J_AYdjlv1>4erqg+uvD^&%r!Yw7AE}{i+}=60ZEBcKCCA#4dY*3ok>qMfu-l z{?R_=A5mT1pgl^Uk)C|Z`f%s1!fLF=dThjIlq0|Y{Bh+U8Fb+f_W$?1n{jNMzCN@} zXB$5K`}wDtHTtKpU0PLmyg&bC{Zok6&F>Vy8+%dVU2NT?{I4^1C0wtqiT0Q}NI#7H zcSCouBMz`55EB=3{!ucH8tEm;6sjlNLu}31uzlBGh00p(sQ0V#z+Z=|w9}R8J&)}c-d_3$P`SI|bK_3mf27NqyZ`~)u-m9MsImh~5|NHR$o8~-R z9Ti@m{mHOz*r@Qs=-Sj_CmlbHGwAu$N5dO8weO$$csM+|H2m}N(s1a&@bJd7!+jUd zI9m76aA4}t@Pp=|VgGA`!}se3hkbZ`!(WHJQzuyeHX-b3o)Eq}d3@NN_{-4y52fM% zH7?NeHGA?aW9R(G)|>1KWNZ6}3&NLh1<~{Gd!{t>(yyWC$M$P5sL=be?<;hf%P+1E zw@~p0ADGO(#HS@6px^)P_($~r6^;u1=>t)M!5E6+D8(3z#n$QU^#SS!?cR9d*j?k8 z?6M;h`-L~(R{mzM3(W;Dhhqa?4yn;wLQBCvg!GVq2n+nz2~6;t>}S|HPqMGSn1Lhe(;qG5hi?ht$M%-29`aj{16-wN>o-_mFGt&kYG zAvA9Krhb@jhT5y_kDKfd#Ghq+-4WZQ-1%!tLpbto*yEA>#*Nfe$#ledgLfQj`l6` zUnKv<@=tb+lz%dsBakDT4za&Bu)my_B3tIOZI;UaLiwL7&Kkd@bW&(T2CZAH(L*i@ zH(T4cxG0>UpG59}zO8-M_}TNntF9=V78gqwhBM?j)J!c37syMfrq_@3TyGbJE5f~K zJnOk$_MDA7B%){EVGNEP5I@CMCll&~20YsTxKp$@54({Or`mKI{{y!f->U+%Z z|8eMHA4J|y5ZAoW7$l;;g=2{N9$UQsZP)oAWNNy-$)z(8B^ZjR?;+j6XP`%Y4$<1a z%5UpGc(XX<$Im9}zu)11Q0ko6CVNVeV^MS4UYq0uM12rZ-$NXegr^|?U4bp~c$|HN zt?agK?6+6RzJFyqvI}~DQy6-li1z&`41Is2zW7w3eXE`f)BL^+JrxCE2HE>_<5J`t zRQ#kkq)zcC=xOaov<6@SeG!(RQ(KZrX#YF3|H#sF)%;WbfAdWK=?nZ*w9wOY`KRmn zr)b~CzeOi9=t4KL$YHRwmth4~A?CTPCf8y;YV9*yhx#@A>y>;~{(ORLL~czyHB(*Eds|OQrpS|3GpBAKY&aBZi}hBZ(B+ z(0;`IL1(f1Tjc(ZbbkxnU*y++7rVcw+~14t?|JvP+5JUpo%`!|vE%NmbaFU>lc*6_ z?cCG!xZl_N{Tcc>MAsC3n=w(~S{=I}?h>w`7kPVRviC{7e@*xX`cUDW$)fVJ?2zAv za7Wxd)E;MppMAgok6oYt{?%lCWXeU7eBhjZFX;otK$KuGhGIC{z1tniUsU!o=-Ttp zH_v~6litM7Z(hj%U#hH;`Tey#sDWh?j}L--%U(H@~)WQy%D#<8&& zj|phqr+g~Qlj!yDDqpwF^`lQiqi}LQ|AS0iHMdz@jLk8FoQ0aR#uCVRs2*h8fP8iI z`|qE3u&_Rk5WG9kw zanZbiB-#71r|jUy4te>>kcz@z{%~j`Gsvn#`aY}vqMty;=Y$)~PnoKHK)!#vPWzzk zeEfa-Blgd^u6;r0DecQ!+Lz1Pm*d(OS! z+BZC!hgO@={-w2l-P%90K{&xrZ)C3|`RwPUa{-n52%~R`U7}w>FRr0|jq&{={R0L1 z2ei#y+U2Ni&Wk_y3<}bI~99 z{}1T>)+ndd^6nmWH$U|M`-_XMGuMGELFE4rCWoSW`S1Pz;libezFio7!!Ymv-)8?m z&HkV4IkU^-WW%$b>z?1Q{~6;qV=*2RFbS`A>odnzwDb8p672tK_Wv68{|5FyJ1k4) zg#YaNpKiWynh#uOPfqDgLm4Wz>cbx4`x^Ah-@a@-&e(Q?@*+GB3$O@Funa2@%|BU1 zuEzJgduz#7Sf093s5a)|uxUg6%)_6yg@8__Y`Lh429Y{`3gfPVjv{4eKYiSJL( zufG^=-Wa_EkJtZr?ot2yU~xk+98o<`N>-CK33bF*UJjL8UAJqff7knQ)%(%o{kYCQ zRCgpX*7-gBgo?uQ5Un5R{kOjecX}zL;dPV*8iu&o5(6sWE(7fiAaBR~nA+_$6(6aayee17;R_C>) zUkSAfUkP=~UkUYyqX7vtB8f8plX72XM9{}z7Up0cYNc6+dc@JNr`-EruK&GU|9g2@ z;G9KRf@N5NRalL+Xx~xJ{x4_$m+OBo*Z*Fw|Epa8S9!=SE%*MH>whoT|6Z>DyhPP%YN5h9!r0puf5{-qwxjz1^N5r+**3weCN^QTg1EH3HQ74uIGDGd#y}W zeNLO;w}%nKQN)qNAM8KFPxzDhKl&A#5ApxW)Lj1mLht5M?GqX8Ka;w=C8VU&h759O zeRoSp<3tpytOc_2Cxo;PL#-sUh4Get>8XqW){yA1J|K48?FnYmQ3ED*aQH z?-YhH^uBMK$4rioj=xkGdOv4QGQH<3))%Q`SzrMcOb#}WhG9etTO^({x zXzYB3^JZZV=3xOAVF}uc&Hosr?-7sJSD*L0%i4FcNj=bvW1HOTb?*OS_aCiO-T%qr zp0#I>bXH&$R^!q9p|$kLX79vC`eyv``uz`n|5_Us%AM2qPI>6-^8V6$KB8XuH~jl6 z{E6~V6~%q&m9UfS{q!rLAGscJzG6M{^V-Oyb}{-^Or<)r>ND!;9roLCUNyOw+>aW1 z^!=WL^uvhZC?0*kCr;0FvmKD6rx1+^WVdNQ$*z6czj6A%Q9qS^fXCn8UFtXd)3zv$ z=yy1QlQ@ktIEVJ@+JAK3)&8T4-i_>1?Z5W@v3*m(*B!~ek+(G8IQrIM+eP#Ln$?fb zzHk4Hw1l)?@ni4ADYnURHVWBr-v9i}Gn9{B$FHHajy*y4 zA=-y9zb-`mU#V}m>PhbfJz6&wt-b5UL<)PJ^-+>CN;M-_HrH}+ybqP@%y zl7~?{SNT`J)gxXfPaX1xMtTwzFBONF^Nu2pBvNQY_FKiFeU0*8q>QQu9{Yd7S>YVn zbkX=HjyW&2(*57!{*$d}TWJ4(bFq59%;#^3`T+Ysoc}j~{i6QM_>UY;;xx|S|I;3v zBm2m{Zz!K%cmIC*WQeJgFNm++^r3Kxyn>o_>Rs|0qO}zDNZe&Z-1Kf;XG4%D99w`J z=)*1C!96@c?Lz+XVm>HeHEvwHA;I_D#|JfToxII|*u%fYF{JR=zisBg13-rtezr|4caO4*1dqDTfhjZ;q#l`0O zP9ZrKHS3-VdAPA_{-vt%O9e>2DZNY zupjqT_iBkINPkMrmmWwldy0eum2!dYdwo9t>fuUK5{MeE1O6{snCDy$+` zqgp=eh2!$DR(L%cuX^^v4Y%2m_t1f z@_)9!L(|**|7!j}e?Em4dOD$ev+>)Ix7P=G-?3kMeZNsQ$-~I%&*}Y`GN?W8+3H&- z;^Ii6>R+_?&lQGD0srDfX^U$^Zh+@3?(z8LO7%t*j}thFR&nVK<`UD-;2bU>_L{Oo zRv)nbiGGD%VO+GAtly!2cvt;!H!4Hw2YT|VdBu+1K=unoq2f!{-{wLI9L&Xh8DaIhbKl3+=!dUuvOu!`cz5G<@tM&eD@Gc5Z zLm6hEb)t7{NI{rI&y4i`(C5(SA=kqnCo3euD~kP?$SPM-|7!( zpRQ`3?z0ijYM+oqQ?veq4*dtS`O;`v%)d7OuNBevbsvBKXR7uY`S1UX({2vaJ|o)y zf3^Qvi}l!unsxTSS?IfR^f)^pTANr-&%WuM)Gjx6KODnvAh*ufb}#n+cn?$B`yDz! z(p&uh2J<2OrZS9BKe`9c`v0f>|8({NnMKr>U%N;CP>(nozL$rqRXrv>#*tLm^zR?)1Vf^RUo(PAJ7lk)Y4G4$uLu+;p zc0XzC${vm##o_x$)`k5u*M-+-ZV3CD*M;vL*LK2!|&?_tN9zYjmu|8h{f zl5s4D6X^M_{S6l@H&1*rd~blh*Oea*C&l;u&Bv_``-`w2uW$Kq*f)7#sN41NP`l&f zAzttib@j(XV%01G->8=jT;$m$=eFYHlrLh2S$eK zJ@&<<$E^pfM-_c1lADZw8&^#X8WkEQTWc;ZwWc)eCHJFsrL~CUVWh=(8eh#ICVUjx zr;Xbi)9;#S93QnujL)MUaWo)-MkEpUnD&e`e(Tob;5>~wa4I;3l=XP_Rb zojEGhMRP0={a3>lX&+#lz2?8RvHu%9cfY%ZJGh4j==Uw}BH9z|4>m>T8vjG)zduc8 zg>z)nDfT~()p=K{*`M9oh9m3`wCyv0=*1DnFGhsUfho;xuKO7r9BBZPX zZ5cU2{W?PZIznICh)}!S^Fux2Xt?P4q4AXGcf2$Vbd4n#jG-8gQjEb^jK>5-K7MTe z$S{dsy=i2q+4rF^g+2{sc)ULUruO%~_V>2-*SkNz79~XuN^hH>LWmti}?-+ZW&HhLIQti8XtYMS%(bz5hG~46NqOi)jtFac@uEMaM z+=$I+f7bo?96LSN3>o$5b)R?tFT4M5yZ>*q|If4kFB^j-TZE&wxpk!bPxe$8|9{DR z0CNX=zG{DeX>Laqc49a7Vm}VzdSv(i#@w7QjS7c_Q~S)ZAde!BBw9QCN3(TU^hf(I z=fBUWFC(|dwaM?J{g-DrCpO>Stz-%{(YP;p0@WMz+wU@dYrOZQ@M$y(C$+~7WP<;{ z^)+(?@#+EXzxMNta~_Q!oTFdBCHzP0|7u^*{?2{a#{bFw11tX`T=CmpT*D3Y;TG=T z9@-c4|IzuP_8ncDwC~8Sd}v=Zy`}&EZTLUB(6#E3N&TT{NvOU+N({F?Fgn_8J z%KyJ@uQmFk^?!ruL-8l;|Maov=U1pdhC5!0X#HQ*|2KwS`4fIW`(rG9JSJcgreGTK z>;HD?|39REcOSc4{hUCfGMhZE{J&|Qwex0R7Up0c7GM$DH+lci`8I#?EjHa{wjG&8 zG}pUH9ny?rvz325?th}UMR7xXKSnysumY>lI@S9#(LO%(%uMYE*3#D_cS!#qy3vIl zFI&T;t=Z_0CQyB&}AAly$@)w1uq z>_<&Mj2Mn0jwDiOLk2mVz)AFeT^&wF?e7^f>I*zaUce<>K`*Z125Mha{!zclb)tcu zz@ze?ZFn;DIqw$k;Bo(Co&HJT_WADrV)uW!@{g`z%0C|a|9`IhuT%cfhTfk><8{XR z|3M#8m+wtWv*Hu>@@wbUKNqckE(!hr-g}7>M00-zlS3o@TlUB#OECt~{Nu6Yqxr|< z=@T#s(KE~}^~}g__CS`5<{$6%f7M&8m$rv>Kj%%sG?XD`ZGOW(>%k9M4^Ae>nV&Cy z7V3rLyUaPbYM(ygd02o&Xcd>f{Zv>&&m7^u8~a{HUx8dT|DB)R-OPU{YlrZ^pXR6X zgX3hwbpAKlD4Zno{^9%f&nmxPjkTyS7PFpgcTFAWoT`5TUG#2b?;Gd8ZT-Pg{cb)wdF)Va3&_ZOzh zziUq{lz-P9&5t}!##o?rMPU8&Dq2|2(_xTYQ=<)mJZrm;om*`iJd|KH+;<9%GTbC-o>cCf5^1tTu zz2D$ln-9}E%vhQ8qxCyC$o%)W^yzhp??(2bv3}IaRSV|<9~8)z`z7tPCWJj>TZ z-v9labKjg4`lNFUcW@66(7w+619T#TuJh_YbzqjvsS}&h`X8iuY@0p;w9I#Xh{h<} z(5Ig1w^7}Xfv7P4SwarRP-LGe3d6}#j6v_`+4p46XRHk%W6IP7auTLs8fw(3QQcQY zkJdg%YoKS)XCZms{X@bz4e0gW^#pwYPw@Y-_5S*<+yjSXWXeKT_IOICf`tskI^%(p-M%lvm&yS3~5 z)cp5#$mFp8VS(S2V>_y_6T7h&`_azN?$F+L7HfZtw7(;@zhoA<$@<^5vCTNfCr>%I zg}<0K|E(2mh&^lkKstvJ!%@_Ti@s|Yr$^r$sz;KZLUJGfzmET}AE1H!VExeMcm_FC ze8oHVwGd9wPonZy+Tkzh^P!)?Ib6Uc^cEI|D`ZbgQRpSF;RgC}3wLl2576%)#2aVo zPYy)tU3>G8gE17t5q*a_xlN`ct#rL08XK|-&g*n`FMQ^J-`2ywk5y+&;b1pIHs(p-qQZF zOVecZji@%V^3CT$wD0;fX~oRxDcTtKFdm_I`R;Gs;n` zovK^-0sCCrl~AS|o7E5c5u4N%&6kyLGWD+VeOvjyuY8kj!tLsq{QM;6X4EhF?+=q% zMBg8dzMs9_|5RZob|YVYubOi}e^h?=(+?s!Ox_2%2C`u^`{hN?2l@G#jvtMF6P<@> z9dX4=`YvC0?}dAf0kx4CRDRWc|FSi{^b~H(xc3|O zBPOpzI<6sg)|_Fo54Ug!(b!J3|Mxw5rrW!L2lRfMynh>{iAUd$tnHp2>Z~WOANG8R z)|xj=d_E-RKOY*AoXfWJyARIq=`W3eD8XQ~Ti4!k`T5XEW@eXpr_1#3TVGGly-;TS z&H7~=lXeO%H?3brtM%<|?@E7)LYm0_a-fI#-s3S>onk z9u}bY>-Je7d&2avgzWp>^stOvfmK+IwOEghh`Dc%`w!MxCoZmLwCf*doj5%%ul4d- z@oD!R$>Xly{Yvz>{_D?&T6wKQJ>plziI0voN;ir8_v!%^fu{XYugWD+T~p?#t9wOF}cqkOGYzP2dmWL7vwHl_K$(mb|}|BIG&$|qXU zhG_j?Mmjm1K=d7;)`k3!$>#S(?=PFYbDjAIj-8IqOG_J%_OCf3E?WP0j=X>xW%Ckw z1=W66FP->w?X2)MG~QHipuu?w)Yi#A>N}i=26_UGQ#~teRZl*ue{cAGAF}O5;SPBZ z?bp@6=wx$ccBs=2DO=k-Yvj`Gui5;sh5WCj>HzYO_kZmBm~XsEC%^vT=S894e|HbQ zs(m5*e)@bENcNB=WbbF~QJ@YP9L2F4hLWvYJ`{$NrATiuPD`d%@&m~6$oE4mcU_lV z*Ll}@(RHf#Cpb0;d*4nwGLZMzcSVn$G@^W*V&p z><^@N_{M>B_TwN9qg5WGHTp4n=BoDlfc|%S9JzPRUqJVD^A~Pwqs67rhD>xECvXyd z-z^NM$)2C<7a@D!(O)u7dlC6F#=ElmtD^7pUT`d5|M5RA(Q78Nf85_I^r%my-WWkI z{TiaZ3!^;@^8NoC%ry{~Qor|+w-D_sd564*G(EpBkazEaaKFu-hkKJlejO@VtBlk| zF&?qoY-zH3qVYFOqEEpzL~{`0LtG!5BhmbR`%gZacNM*F zWzMVoHNW}m>WGg#8DNw|^yr6bbFBBP-jV0n==2;a z!xN!OA4BR;ahM@|0w-}At^B`qx;UJnXYSg&@TPCP(Jvr3jBjmRx!XBixFjxSe*YD+ z7d3l|!!_~-s{O8BI`MAbcoV*bMm}wlZ`zHqzKQicAXMO>)$f4=@AML*9Q~sy5$2sCl zg)>Ebc#NfwL5`k9w|CxT{(Hx8htPq!)Xw08njaKj6 zT5>(oY{O_xZ1fG7jl!FerRNgf(`N4}c3=KW`@t87a>uqKT0dV!?nE>{KEHlmd$LMQ6}7rlDf`on^0)<*DIe_}q(pt4XktjzkuX<^6G{}%p{ z-L&)RNg?TXDg0B%ZM341RN2-%+w3vG^nzq#1nnkC`&O#{Qe zo-e4!4r1yNWB%CIrwf$$vbGU#@xPs;vh8n{f8d7yP3XT>0 zsvuqPPC?6{8wGve(O=+r-CM@c3P$K-HGaA$8haTW8tn0zyf`#877Y$ZZknUuyx6J! z;YY{yOB8$}{CK5rwOi*{b9cD;XXfWG)z80>f47Ezw@JS~8tKX9=KA^V4fNp_^8R+S z=Y>1M?XMMu4s;$ehG3katHatbWcjc;G+iwU&F@-&N2c!k{u@2bw`@J{`)}R8|3<$j zod>9}#<1TO@8Dhb$IYV9pB{I8^`q^p^aD1qzVPHQ*CsB3hKWUCpmS2o`QYSWw2rnX zE;$@&ZD^;ygUqh}p;UMbvh>_K{S^8Uy6IWhn!}^(?xbhXjt)HfPF>3u*S$eLHpvH? zhRBz=#-;94bl+yWCVGAKheMpK?f!76OMEzt^`GN00h2HV(@=&P$bVmf-BbT8`)4-$ zXD<84^J#2mTVR%R=3pKcU=fyJ8CIZuq4gQ&NOY2!b!-y0L-$SX4{~Tq@c(!5|JgaI zL*_ry)6Q#UJG8MQV%3FzF#mCtG;6LFhU&Kp!)khbwmiM4Ux~gE$+SF+OQ6BHQRUao zuXw_D>Kse$(DzS9-w=XD`FjpgbU)N0@B#O3GX&1Zi!*hkmz z_hLVy{qGNwhY>@UavYWMsBA~y{QBemz?MDi4Lq7})I@K_F{E%*8gV3%LK~ue12bf= zHX-}B#zE;P(D(1wh$M=m2^{`M}_2Md&|EQ!fNNP#d>VSW|U()+RqyQzij;fUE}{(|9{Hv zKgO!8O!WS$p|I^sW697A-GhJ~B*-8a3B!v7>9WIgkYI-)NU*~Y>>$B@v4b5X*g-6Q z6jd_FAR;2t4C1khsybD5syKB{{Wx{bksx6@NRTiD2}7__Ly*gm8U`7LL7Mlo4h^?k z@4a*1KfZaMz0Th2?6vn^>sinGssF!E|33;SqRo8$_Jw1@QKZpvOaDKz=tP75{KlRB zUs{rJO#`yUacj5#&x{GR{$mGrVGs7<01hF6F4uIo=;yB+6LRsHs@uh9TUy^mov`h! z|Ho0J(IL)EeoV-covup{9TSd-Cy5l&$O_A#lb#zPZsh32i1$)>bBs3DF`*Y*JpaVJ zF=5kU??D@UJ-zUDSvcw3I(pMW=>nS`YQA7z(RAtZY@6d6256A?8rk!vmFm|U+4a|B z{n|H*6som#o)Xs?oWliN!WH!65A$2^dHGR$T!Ac}R{uY%{)?kX;}6^4(>K{C@!UXR zQ+XI5vs3y1eSZ>e({m&E|F}!PkJx5cKwci|#Y15U{`w$OFrGsFI%@%u6{xHGpXJDK z$0PB$U!nOW=|sat>3>20E0SnN3tExFDA$a|IK=wT1acClU?j5eB|ICM+SBVmY?`>+i}|kgL%D6@3C^EXUT6ab1zM-IZIC>g)|19DG!}wLi{}G{bv)|R3{9nw+$=4qF|1}?0j%qK6YZAsi3|iRlxEAtA*Q9+%qsXy{ ze&slF0^&HIo;Q7$Zz=yBPeEa-dckbp=|bNr{;2<3SDI1i}_f9 z53YZ$uDwp)RJV`{#WZ?FA~QREJfmhxiDnibl~2lr$Lj{S zzUBVp^)-H5i}k2BcA(V$^tA79rnXpNRj9=d6n^te*hTI^&6n(J@QnV)WAguL<-`f` zD+`+6k$(448eil&?Q`w{96|y|kVFcFPd^j7PRRd!tS95YZ+WgLjsM=S{P#_?zasyW zX||z*%s7r~^mRUv|IP1Di>sfl?<8|5;y6yCy20A{1`9bmYyer*;b$-duP%bpyIahQNfn1X4Tfms;%@-v~5 zoP)$GL&IEhJ{DjR>No0te#t%$^wR!&oB6-=u{-vkuH+|3kW%c=2 z|C2UH=Vpun=sBxh0Ht#DpJKVJU0|hl0=|F#E$jd3&u_e`k6&MY63u++7PKOTRpMEV zHCT)F*nlc*MHfG`yU#O14!u`Bo9mvDzWpNqyltWJ2RMp2PN3rfpVc|po5mlI32hX$ z;@N>+*n|3Z^Z)s_`{+q=G~odK5K=F=x0&v7w)ATUNC=DTJ{*bXB70965yOx<=DTc^ z|CJA&j&mrj`>XmBCa`(lQ@OJeM~KUwcSpP~;z1>)YNPx?pd{6A8A zt%2sZkr;)s7>9oSSQE&;X5}t9!2V4kr=jFOUzPuE>c5TWd|4aE=ZpjTjCQ+kek9Bi zR*Aw7*nINw`2YFz1z3cE-y8dH{QnYqS{c5S94_xJCs!alR2lQM^$O^@K5gUAs}IoE zpm*j9c{+UE#gi}l!mDr`k98rZ8w_Ns|ZO!7yY``DhFY|vHy=?VTl+t>cG z_H}-1nyu|1quq^mxO24e|7>qJd)q_iX0yB6-t!~)-NV>nB!<~%*8l9n9_&N?EA0O~ z;{xc(!+hgcKNJqp6G-iTum8W1udLtT{r&$(;`#9$#Qh@*NYgt}>VNpIJ_33X$8i!3 z2j5E{8=aJvW@#hF81Zr z{Ppk4VtYa1FXa*U)3B2LUB~`zl)uR4H{|b(_WitmI&zY8reGRoU=}Jd2XoQ2@V)*= zeF-`A4wKJEi+_ZC|FrrCALRY}{gn0tdh>twKh77&0>m<75t+F;C@dkDB0F1sgj|8# z^UAm~W!uxrII@^m#*wT1wi?yHcu-dO`R~it(AQ!;2H2quWbFU1BKxghv6YN@e={~A z+W$W7@})B12l@c~z5~0k4>g-Suiri>J3v2#1dd=p-Ecs^QewI`O~)yu(TVyy+WE$N z9`xi{&sLqLNI#BPXKHoaa_muEXOFOxIE6D9_}PQ9bL0hFLjN}(lwBeFKKJ{we)1Y_ zU;wv~zWQ$2``bV63d;)1+@0LBJSO2qj^7_NFg0TF$hh@EQJuGW@ z@NQY-_;<^iCcj&jB%5cyTh>ChI!=*=PiYgZ_8h*W{;MtEDf$ppU^u!qKP>A;&;EyH zIrMH9cgBCTKP)S@JS=Nl_HJ4G)rVzAZ$2zb_dP7@pl9gWix0~>k*IrEHc~vJFc#xb zFRV@+6X?ko+?#kO(Wf9a(Y*<4neJX^zFRg;So-?AWi!ZG$nJf&tdg9AjPRcJcgu3! z@0QJVJRhZfVUz*AyWcI#qhYl4OKX#~CZ)ZZY+2|(kir7jEW#2j#d55`Dy&A=Mdc5A zZYqEJ#EU!%D57nu^t;caW$%`y(IL-glmS_?^LiZr_n>TzcnV+8&q{7U{n-a)b;7IY z$p;V0n(jO(+e)uRNEWJnW*^e2)G8 zqOmmeeK>$aNT8stzCm1#h-2hR_4~cX{~eV7$&_;Ai1U(2A&qKdWjo0nis;h*-;Fp2 zFo#}k`gs)SMYPG+?aF|oC)k1({g2(+1Cgb7)(r~B#c>j+a0cga0Rz9)en&GXjA)!p0LDrqO-x^to zdior4E|T(l(=G3+&%0opSrF7|$R5r`g6Ix-O1eSwJqr z5-dgE7o7Xg);#@2Ih$`E5@Drz+`m!prpLMU8UE=idX9bXdDK91evG2WW)?)*zuobn~fnDfY$N%2wKgir^^^5cD7JF78i;mma z-ga>xJ;=`@OY7gUo6*L0A~E_Y{W=U8m&hsG|{|jks zX_Utf4=Rl{`>B41#Na;(aZkF#d2u9+A{$!b8i4v3PV0ZVtWSY#d7zKs zj((NV)(vz{Kd#{h25=j9aUWgl^wVtA|Mawe8}zy+U#9;F#o794-qcTnqe!D;p?(}> z(fNw7m&?OL@eHcbeuyE+ZY&QO;T7~;rd-=jc^FO~iQ?XJ?T_VrCv_x@5*GKa8%vHu z{Y-6yFbel-0HZ6Y~HQCWkf~TJh~!m zc|l(~-*kg>9*?iDqHjeldXDjr(Th9^D59Z7|36>2i7%Yw8#j|J{N(84rm(}cyRZlQ z5c^n4;FH`&qt0g-^=IE@74dmo1c%idHio2eU<->4tfSz=XKtx2#Gf; z!Xfd*wegOSN!06muM?h%$2+wRpj7|(nz}Q+6)k8++Bxay%ab|8@&85gI5K_O1v1i| z|42CL_!J6kf1H>1Wc>fI_vSymApO$YJX!h|OaC(I*Je=N5W*SPox=rO!WH!68oJp1 zZvK1^O6xmj*#C9xfBYY^ZMyVpD>(YJ^rM6Ak9$L8(RrTzZyXwKh-Uz|aTlfW|8+yd zeR?vs4HzHrkUpr^`+tGmV7s1-{~zjoXzPvk@F~9!K?UO8@ipI)SN^3ujC4E-V=)dB zFbPvonwQ}oOY2aM_g#+ea(f z_Jw8oTU~PsXK)S|a0yq?k0+FBuQ(=H!(59Ml-0HZ6Y~~-|79Qt4mgW~2m!I|i?+Uw* zjJBDcS9}-ajlXj|=sV(|7bp8prut6M)}np*g!bWM+5*r)&zu&|E%_5&W%4(Aki#GD z-}aO^hoAz(k&sVElA|yd4X=2<+E1FKJvrU;mH%6=>fb^t=Ku49!#L+nz$8pT{b2s1 z{4kB4d`;f){m!7zLh5ySL)hc>e=3E|!CcJ80`yt0Vi8#y@2D@R)E`6+{O&_xiSVUZ zj)FN{)xY3F)6;$QpR51y=T|#kgDg9>mRyg=<8|^G?N;jbMKbPju)%MA(Z449|HgOz zb^Yn==T^tH*nwTxgMB!FLr7rYoBIFcfkt_uNt$9>o6kxcS`o)TA8}q1DWuVf9Ev!O zuGhUE^gQkT;K}?ydU3G#n|W{i9~svQ?0C!jMV8+An)d%+@cnZ-`@{5u0x)O_A^(chl( z{$m>u_bxQQApP6)fA7{$jHLePX8qDF=kP7k$#;Zb6pmJU>e!eMtp77doHf?AsQreuRp|5aU#|c2^8ca#=bk+8{uYR5 z5td*nveK22zUB1XcHh=!`vA~aq3F2aIL`6w-K||$*cv3(*~fvbdwxtL;m#pz%ygRak8&@OgKsQ|Ld4=iadjJxPVKzf__}Xz!%LIj1sre|1*9x z8Q1^0Pd>z;zx54b2;!bX736S?#3*EiWfof(ke++tQN6eD6ZLY(#fiQHVfpFm!87&$ z&6NMqv{U}yCjYaaEo3XAk6c>+>5j3snB@8zW4de02Zt&2Y3TBPyOr@hWNw*vGVeY6 zFOL^dn*U$N{D*{ss5oY!5_2#Y^RWN}-!cDN{i*MZW5OcG{a>_~ z1i2K;u>y&gm7nBltif8;3#(JEt*0mD@uqq9RijrSHBs7xwM>^*=`QWB%QiIeg_Ccv z589eb`|HwEE6rPT%~sT62j1WRv&(VU)%Vs1zQ@j^_d2_dI7Xnz-)5I<&mG&WV?ZW|Nofr4NJ4DaDpbp`j()hn``mV-1f2RMK>c7Z@a}xgR z2$@72JCGvNDD8jew>XBr({T>18~y&8bds?Tz2T<$2kI$J;!C1=w{Wx~g`#VY<0MYu z49?*KE}?6i^zU?!WZbK;mmc>9DyZL--v2G>$5EtDNdGZuLl&J#cPq!m(~oPof$T8n zJa651dXCNR!EO3o#5Mg3$UCQZynd3G^b-hwi28XSD%Hg|$%~0Tdn1mppN_DY*0|?k zE2{renIjLycin`-FRg#S!}Ho}+zgqyX#OiX99j0SW|Q&spV78O+%zrW6h^li4Fl?_h$W*bJ!kAr=)ND8ZN zcqYto{#?vQp?7FlKrX@(^u8i5%j*Ti`9@vWwFjbSnR^!2@s6?p*^Aoy(DpogjHC24 zmWreQpUqVve0ne<8+=|3y>tm8Nzm4+_>_1Mg-*5ci zZsQi|N&a{f_R#ks_Nm6S#xal0I3Vm05;%e+Qb?n;|Ccoo`o8O1aUA#m$&p1I$4R8s z;ZBifa1Iwxn*X<7`wIQ>{J$&oYk0DL=s{%_8V2)=_=QcQ`JW^BMSRW{wDOTtSnHZC zr}_ANUY&lk=w=BF*>r@g}e=F_(0iU0XLzYf`< z{Je|$ulZmD;=7HzxR3g?`oI15kRICvnlR|Q%6z0QyEn8TuGKe0SlWG6ki(JPY&<_X z3Yq=yt^dE$SboRj@TC8LBmZ%o{(pXE<8=QqRr=AqSlloB55%?hC%9%3reGRoU=}Jd z2c`7i`GfTD*1mxv+N87ns`Psi_Tl;Qj}5#aVws`lW3?J*izFuK&M)Uf^Fm z>Hm-SI#VC8@Hplt!~QIy$2n$8$(nC^KjwfGXR0soGxFoTZ{+B4P0~T0?Q-X>z$#Ri zYYWm zum}4v@JaR6Pa4la@B1syJ|oY+ZjMNqJpZ(~$vDO*HPwI6MO5=Z&b4_=oZ)ofnRYw}EZiG1hFXums zIEhm@gLAlmODL`XGut!0>D;TH@dMAAEI2NbZIjjirt|NdlO{VHzkmIoO~T`tzqtSD zfV~E4+RR`0wRTT&4`>@G?f<83px<$v4{?o*{?3hfti2%a&on^4jk~yy>JD<&(!X8$U-A7d_WdpM{gIF6 zpK2$%WiC3>>wNd(OCgO;WHa)~O6^AU-0nwhNJaW_6sOBq%8C4DZAj!vVQ~-2Q)IPq zFZH*q2~IzUxCdpEcCO@k<9r-nLaXDHzN%)jK8Y~WgQ%k@fYE>(t2w<<&8 z=@-MUvQLEJ!q;O1O8d_Xuc9Yke=#(zd@*dL*CMs-MfH~#L(6L~hGxJ0{cE2Iht_>2 zynXC1!@#a|R{clbUKYU?w*tczJ_`w^W3VT<6Dt!N~Pli2plf(BG zej@Cqw-0?bv<-eXq|Hk@y7*abBhQ77%yVIf|JsE;*oOl+grA=MtZ`ap;V0{6>i_y| zIQ)*fuJG6g^HX`gZgjczAIyc9|J9CKl-Tu>-@0Ldhw&y zZ5SW6P97hQh$D%bW1kAO!zPF7(G$bA5fj7TPMmCC=}(1}-_rQI(Vq^xr1iTOKOJ_C zo*KUY+VrsJ^z_i_w;X=>+RU(T|7XGv4u2->-ROC|Hai>`Y@G+c9mh$W!rRKKGvqm3 zz$IKk!gttDUc(JE?0!-EjCXMJ#gOdt%+Ye)J2?BI_9^rG7nXYvX?cgNoL7)2k8;$?f= zBY)Li_ZTN^0urZ%|Drt99h)5{Ii7-g`ZRI|lCx)P%b6WoCeIGdNR5~sT9JKSf4AQ& zk)h8a)AOtgNX|#xOQ?w6W9kOzxu>px1%6wEhRN0+GTxxH{!oi@DP#OWUON$)x~P3a zS@5`jV~O*ZVmWH96;tzedFWF1b)#px@=}@hf$@*;sC&4lb{u_4S%Hq3%2s5Dse@pZ zxcdL{#ju*}Yko1TAqT#t-IH984T$@XSCM_+k`Kv&-^%Oc4(!4nB=|b}$hvj*Lm>~L zo}M6&AbC|;jF!vlS9sk2-@wmoWb5J>`{ZnPX*|2MjQwBCPN@g1h}R;8G&+$(5yw&T z|2MOdDEYtJ*+vu&^MBDM&i0}F-@$y{oBZE9Y-`_p{{LlRjr`QfkDl?S(e_LiXI6L| zBY%pXdq>>t>=}Bs@dYLSpUuwCWPjP#xOaF$ytjRy=bTf&-<$yQ66&r?S6;eyYCrhA z`taA-7Ph+N|L>K~jqI=cZuEPT>q_JQ>CrBi=HIpO&BfD?Yq)^{+{RtpM;BY({hBf! zIrJir$LoBxU6g(tmCl&f4)>GsT(amqC~TOqec~DPJ^2blke%)v&#Qu-8!i19P9KTl zN@;#e`Zr2HMhQzSRu?13A+DV~ft-ZcHW0^7CNahFG_=l^{ujLGnbL^{wz<*w&@|CF zGBnTh{iBth!VK5ULM7&6J{F+RrN5c%y2;<(CSPrquMqdd%ky^&?cNpIUR4LjQRk)A z$2%VI$H^=@&&yxz@@8y7L0iKTamG0iOUdOZjVt+`aV7LsSdBGUi)#L7>^E3X-+(G? zMfRTmxTU^H&+*TDu!Ft}#n;^1TkcUEB#sr@BWxcIATil|X0lG5x?X-x(2pRACV4f9 zKU^Q7{}1}#bdJxOgl{ z3Ow-#U-GRAf8zhne$W5i%>UiV|J}y_&G0Y8Iox$4F$(W*{~PPL>mdKXh5uW}|L5=b z^8NG6`2X|x|1a_XPxF7z^MCoXrTy>3naT72$xhcL$e4%6iE9ETp?)a;_vw#>DfHwp z{x7{W|HT-r)LZWDP4`&F{}n#XIWsT|=}bkaBs1mKH~_c@N|qQ_YO9D4QbKe<2m z*zh_Z6iwnxBF-sjK`T<2@7e`ege6#tAxlY zNT1;UACrD$(TT)8eZJyZgSA+X`eD-lys;GY-3IH!61M?PSmYW=s~kc+TJ)F5vO_ zxl8m|ru49zSLppH?d!Q;{@v}r$ZNvl`ad_w0n}?dxJ}+g>;o&cBc2!(?mK>nR%JoT zaWfg$8GP#d;{7Gxs$5^Z_Py;dfn%}4g z*|T6AeFFN;ftf`1{iY&JAqSo{MuD7xS*XMu%*A{xz#=TcQY^;`tU_%6+gARk#_0J* z9LF^RO6$k4ld*hRBP=~tJ3m?dOLO|jDr6_BWAMGU(&L&WaSe!C`VN%V|B}Z1G5Mct zxNiIvn(pb#M>D+zt!zUIgIu!~rSUI8nGqY*ZinkKWyE(?K?heT{ql zw9RAxaC8}4$4+&;!nWOF|ImrVo5mT4CxIhKqW*Q`4_-6=D;`JNj5NIyrG1IS+p?Jr zBXh!vNb8e7PG-oHk`piqT{qQ#zBUzy`7WZmCZvKF~x(s)POhRHV-eR}G&+$(seMNqPt4Cn$AvF_EL4BV+Wy*Z z2aE~s)5g>Hd-ngc*7I@wN#~qG&5!iw|L$WU@w#WNpY@EebGU#wcQTfJm*~l-J-^w? zPkKL6zJ<7VVBBLqu7TFDdvs{D-hb2K(IJWEv!i`mqeJWY(INHD=&;B&$8ileFo4^* zi~Hz$%lvoroHj@Ogg*Jp`sT?3ig(6@w(ilP-Tnbbw;AL5ivIt_`nH$p|0g@=jR{?+ zjUPB~{D3+9IkMNB{QLvs2d*1GfVMj8&$nBD9_gLd@84_w{vGr8ozqD+=%XVBk|}XMbdQ66AU|LTD$p;V3@1k-eRXUYMUKTdOhERg|G*@AZo0I+V4MMc z8j5d9n|}4Y|LpZ2TVB`xaCmIk^s4V~q51cXW5W#R%|ac$X{GyeO=%w2zj*f29oIEz ze$9P)4h>V?Kbl^09!m4_J(pJ3q);iIIhc$2Sb#-Xf~Dx1sQ(!~Gv)K?zRk(<^;vak z#JOHi*8lgu)96sA;?bTP()U}KCUr%Jcg-n#J<3j^i%QoKF>ye%inr5|1BZk zpAmiz7jOyn-SSVnJVSq6{`AwYA+8A!*9L5vD*uo-gbhT#GB(^M@8UilV$fdsJGLLh z^o`J_=(z7U9}7ds{_l(p6=Z48)$eNa|Gf5uPnCy}!bV}>XV&=t-I&l9@YnVS_>^@z z+Qx*jejA4gn1m^qhU)6_5ce0HLCaAogYCgyB|8fXB=)157`)~k<@MQm=ZvTVkM(IbZXOKd|IY*F03TbpA zha$Q>^KO*p*^|8&AJwf2>Q_a+*^~W|wzL1}SSbG?i%!J;hvVWoiBmX($NdlI=(&w- z_&RMG^h+r2Wp|MGoO*FZSnPl3C$Ay)KinV(Q0jjWe_U6%y4iKcFFxKkw)uJKM8jqE z|C?+Cd;CZ9YiAD%cg1rb4^jBaGhxsVm8BSh3UsOKccbT``u|yV@_Xv$DC}4N-^>21 z^P4=feluWx2K8Q0%f70>&+zM6l~4)C*~ zVYTBmSc~=OQTNKBt8Q@UHus^R?o~uDJ+E$dRCv0@JceQN5wi5ogXS@0%wwR(y`kb7 z5e*0BQ{6`hGq82-_3x&34!XC2W1^zFZkevOp!xqvCk=gLS92D|L>Ikz0!YBI;HoI_TTj%x5aZ8_wf+fb<%&;n1_ARzfk(;eIz_Z zAA;h3aqspYFH67h3So(M>oAZbQ9tz~VH7zQb)I$8Q0W?Be?-R<(0X3KqI+x>9@|hF z7cCu@y{LdHfL9R~T%*&(e*q4p$ z3)yyE{>M>yJ$+05zbKC*+b#cRR{Wm6GWIbMx5Sc}7rgThZ9m?v^g&&iZG)c+HCLYt zwYQ!P$NhE^Tc&<2Y#s5j@OEQ)IOX^Z&fx+s;m2poteqeo%y;naReu#aU4~Hv$ z>&N$BRsP`c?hvjyPLzEs{B*zmyY6p=pY8ruII{nnp}ucpXx#i(XgK`Mklg-_&{Xyf z-|W{z^Q+$oscl~mtp~pz+WWp1+RDDB|M08f=;*KU4Za#Wa6_B}xQ)BGkB1oaBlZhJ zP=SWo#vf=0XhKrEK=WnwiW9~kAT?UMyK~;}|BrM$3S&{wAK%4>cB5yawg_zoy)W?p zQ9u!G%HVb!MH;2{kLk+s+5CTP1L@IEg$d%S`J{S^xjvKVaqq;8y3rJRE+cMCqt8I` zmbld;O6xDxeJIQlmU#84P)W`~y*k%iaz5&YNtg4JSm1aOT2HeN;w!D=(8q2x^8eYA zrab?jpV&NJ``<$Ce@J18YnEa;R-pQ`)^H|QV-33M`2Q_@1TvTB6Oehw1+wJ-Pv!sP zD1S7K4nFkb{b@Q6@+-u#9vhIJIXG028FlikddXILEq0*ip7_zr56>gcp)IcDSNN@d zGkgEQm}PomB^&ysZ-ts)e=F3!{Vi=+((;-#`L8&(H?H9k_xe8I+(Y=U*8knSO`9MR z&fRj84?-r9LK+_^|FyIIm+Swk^S!_R*C~!1ia3tGPpgl7#hidos*n6az3t;0!%5)- z`mP3^(IIbzK1%8d5lm56i%MJPf;-2IMh;uS4tMort zg$jDj=ahxz?8Eh{kVEgiD)qxEeZW)_%RfXowRiOo~NTKbes?d(3NTXw2l{)03|L-KbmQ{uB$yK3ey#Jr=|LOVB{(pr3 z=L01kRD~)2XBuW;7V2O4w)OkJ9V+R`I{%G1^tniFmj2z+pZT`Fpl^ry!WLi=mS8EC zV+B^BwBC|-saQ{0?RX8U^%WJ`%;lsvv>5;6eKg_8__OEL)1PLiQEE3RH@42X8&HL< zsKpL+@!`6+83VJ^_#d+OApedmI5wX*w7q5g4~{x7ebty3zF&sSqVv2k-_I`b?7==9 zKz6_HM_xHZk9oQW3HlKf+26uUHh3yqOh(^4@w|NHw-o9x%jaY#>MqJ__xy&O<04u& zvcb{zk}d08{}1KixbTxG{9RPNtjPAz!EG)_PXcSXYCMr?oGbo>*|p95v=)T`J?feugZVQf!J5v_qnRD z!g>9l`L_1Q^04Kid~Q#GO?B!;^i}bTiejWRo%{>D$>ruR>{k zmxJne>Ys6am#xAQi>>KQ?m+!3-Z!}ibuVjQ@r)MH_u&8zA%P=EqG3P30!=sBkq7L? zRelA%wEoX!_Kgi{Z(-laIF`MG9&J}?f1g*ZE3?k{yp7u4$=p2c18h|u1$vyHmwuT& z@;_;GB8TiN;y^JT&tv~y(trP9_EF<=u{~bP{`ozwf3szoJY2>Pd)f1%*FUX|)ODv& zM{gSJzK0q=;P@O`>8U&H{0Qmrdue{$V&@^bUHaOE=cTXFc{uKx3%Gjs&b+VKxa|LfBK zymfx)Lr`4C{v-dY^uKA0fUt!4hLa;v|CTWUM)eKz}mce!KelHf6%WM?QLC zv--LEecNL7|7Gg`WSSq)aZmjpS$b!mdilrs#UC{;R9uyqgSnWG0rUA5kbUyQBC_9J zeue)qMu5H)%Mr)&^zbVRD$*+)7yVZOdGv}u{j@PQ!dGJr)*^e^Jzzb(!8>k5X??41 z&xKxU|1mE%HA0`7bE>cvwb+4O*n@rOdY<1t+4=+HANldU7a#fUh12}_6a4l!28Z@_ z{CK2a=C?27x6iizz;u56#KED@oXEZ}viHxj_r2zCc-EU5LpUJrEi?5?S~K7fz4klS zSFldN5qc5>zceS++`+^|@5yl*oyej7!K3<6ksjAHY&!cV;W+&yQXAEW(1PY;+W-01 zjeOLmZTv(&YFtyU)c@bj|IO&1cFifA!8u&OC0s#2y7KzJjj!+F`{&4BzJH!9I4+WH z=lQ>v`M+ez|KH2E^*>p%)BmRVq1VK70|U5?I5!}JyY!rX@Sb5G3HRyKP<)mD&nGVV z|NPg7!V>)8L2oPPQBNO2R-lgFv{<^9@uPk=C=3_YIzqZeODF$1uJhN>F8yda?YtAh z-D?Zkis=8B=6~JNAB>T%k7I6YJFK@!AB!jJKPqo?@_euIIRAis-!K2~mH&52|4ZyI z(&#`2ah`A|((YrNcqU*HrXVXUtmX61MZTIq4_U-Mb^0j2S8v-!yKdr}^5-Y@^7m7bD^m%3&-R$vuYV-40~ zJ-S->|0~r$-cbJ_dpD|okOjv@vh5xDA4ePIMePF}d3o>fBmcg0ukt0Y+OnKNt--iP@gt%^3f;@sGQi$`0<2>RtJ@<<80i97talGznX5-yQ_*4?}kk8uq*Fn~?kI5taj+>4|7xxx0> z&}J&U@T_(1$^Ksr3J=LX?a+e``hI?;p7JGY+tY`j0>d#9qc9fZFaZP2#{d6Ix&1so z5+*s0^Z%xh(=Y?G5Z6qtBtq|kMb|BtxGUv3+le2{G>3n*sPkFHBUjyfle4tnM?`~MF6 zk2ohXEuK!~P((JK!#6)3!`yohPSQ`s-=7z^<2-s1`;`)h)qh^|ju6KVoN->5a-+;h8s-?<zl z4dQ+$)nEHq==-JqFUN8IU%zpz>*;-;R_7-Z;;bUK8JD${tVR8G@4nC21bWgvHLbJH z3w;k#v-y^2S5f>dgkfBNAIip?_bt`|BC*5vh9}n|M$%QKW45!I=c1$Bio|?zs~r8Vfy!R6lrag z9mr_2%r4Wvzi@2mdeiuTw~P;XN&M*D%f1|BUr2LO!V&j#&bN^yQ%Iu|HUGhe{7V0( zHqS=m3YyN^pFVGY`WAZ^h_e+bq@_9M{34FyB(l<-dD&cbdM+bvI72^&;!geloAv)Y zr`P>o5SG|DCR`$~p#HpmTJjp|$~;%;PvVB-0kod*T;K7W`#fjoHh2e(o6U29^?S8% znESHbIOTTZM{(OVcX1yNG3bzb4u+rtU2JSOTiZj%xiGzC-f@8}`o7x^oBPsY4oscz zvzz~~Zkh4?vYua@3o~3CA86lnJPKn`c-B60WOakSG;#{2VFqSlm%NvrHz-unOZ(5V z)pO`sVHwP&=O!u}*xdQ_1t>l*|4x>F(W?z%k+8&0<8{cTh}(>El%|5v`(cs}}!yDY}AuNs3up2R5( zd}mNNL!OIz^Vo2Kyo4+0$2Hu*0B%S5S7X9mvhSb9g!^Rw$Hs(*7_~fN!38p#9*w`T@GM$9IocznHB3fGn-sPZk^($w}^EO1vNUIgOlw zS*XMuG>niwHnU0GN%1z5E#hqTpQ&#DIoKEn=gr3gEJDfu7QTcY_tfcOGndkrqxdTO zyYM~#d!_ji!V)j3!;q^H{qHqoH6Nig{@$~V>+Zz;#nuaNbw9BVQW|GSK3RXkIdPnR z^E%JJ&bw*xj>t{dm9^ONKz{y&_P|eDFZ`d4br4q-x=zUdr{#Y#cUk_YKN;U~Q2qO| z_P_D`58re81^x#z$f6Uaac$~@Tg6q2;t4)B^6Y;c=axP=IP4I<3wsd9|78bj-?}_F z>~maNx0h`{KtF`m&FT#(t)sD9yP>cIpZExwMEy&CC)23gKR9gJ{r`ro=yaSzP4}OL z+Uq0jzx8Kf+x|Zbe|z{(!wwYvb{v1F+}-u+nDE{G!^6&DW5V~}{PVEq>Ys;`ejE7h zC&CYJeJJen?tUO_ulN4riSA8$PWk-|&Y|y<0_~EbP&5_Wh|N!|wKx;d>KThg}O-hkoM+ zs>_Yj|CsS8L%px*-q&n<0r{^>xPrpR?Tbg=z`z%t3N>HS?(t>&;yJ#JySR_)zo`fh z$pP(M1J&=1|7&CSe`EaLSF{Nomj5vX6&Q|@CC=i&CwpuKc3A$=2x~Ng|?0STfX|y z!F>72`nPBDT~S)c&G}E(2RNqh620Ag_k(=+z54h0#_8GC|M#Esu>gyZZ5M~%m(X+9 z`TzU%!+(cg;JYvK&kM+36xS{L0yrn})=+jpU1^hf$$a+Z!d9T}EdSTJ$rkJQJ6?@e zai!3lkq-B9bntZjQhyoR-uSe1Ob=_Evlbm?pVUq>F|4O=Koz#47CW#Dd+^hj;~HA_ ziJ%{Tr7YBI_c>Dbneel+@=&)?zgpR6L-OU>p<&+4(0FyGGIwf7ZJZjKmzm2}_G$Gm z?GNrb&T&Yh`FZz+)}iimvU&32J%j{~Ac+*x=tS3O?Ju5lk8^Wm?`+RTo{xPz#naj! z7JEO-q?0VYpXr{h^B%vS^yZ-U>7aU(gYi6i&ZD?qAO-tVXkLv%8 zFY`Z^si!YiZoH*lj#B%>cK(KI`f&|65cjSbAaCO?x}4L!QXS(B{s;L-`w#O$+Hn+V zbRdH)I&ohd4>9Pc@)d?4j@zjq`>ktG_-c6=P9KRl_Gc717UM7h>Al8CkW(-Nvk=EN zWZKn>=s9WYL23QHpJ@}M7w~xh;0nq`C)VX_4Aj;L!4E9V2Sr4NAb+o>Cr@~`SVLcn6ulKK%J=B&H4J4R*`!iC z@nZIe{c1sLH#>!Cu33)_sKQp%Vh47i>lOBSG5frbeSV#N=F{g_vd?d@&t%&>>_3iP z=I5U0|DI<5*Rk&#*>^IA?GevD9Ka!D?>R>~o}lO4Uk^&{|DTfoWBdQp>~9(WmyCOE zCU*0~UQ%YEK5xvm>yjv~|1*zWMapp+t*_AUs0WZOFROpFyFVN~?f#CrKV;rfKk@%@ zenQt^?fXO30mn>s@_O!j=Id%lSM@XcraFItg8 zZ2v#!ybHL5E67fD4>Ohd^xO#P$2Iy56yH#nQ18gElm55N3lJ9D|8J9bQ7`;H`4FY{ z|Aqb+gA%@Pw7%efC%TW>?!*85QTzWw^=h;tg(0q~K;d7k^+S%rSd7C2{Fm+jQ{_J# zoi6{Wum55D|0el%Gv8zjA0>_foFvZ7dHIv?GKC(;|4btv&)elhY>aQ7>K#yZ9;yC=4JMx`6 zNs5en4Ar=%w!<88=Nv);N8PDdke)i zUN6u~>%B}gevN(wPu3rJ-nTqHw*Q)6KK(`a_M&>~i=o+Z%eEIoD^lI&8@r|-*Kh;x zpZ_=DxXU@+=$Ti^|F7i#SB5+auT+NOOO>Im#r*V)`RVQEr|&dBeZTqX$Zj`39SOh1 z`7gJ{br<(hKlnv$S1*Q#^yHiF?e!NQ<$wOvll2E)b+0R541<2=8SJhMPmx2A_1_ib zaAdkGwM)6Ljg?`f<54KMr+8nb`&{h)_PW1=?yu4P`Ol8S?hjdXqU);k-IP8u_rUyl zdj7Kc^XJW<_YB91a~vjM5~g6d`<_P5K*KW6_l5U8-w~d#cv|mxzT!%VXO{CSF$Z%| zKU(^oGoPNE>Ds9;h6VIRNDX!^T82r#^Wxa@Sm#{gw>ZXNDY+b_`F|fZ?ljK-3&x=e zTa7hXi}l!mKbrri&fkL4Ivwmpm2@Pd|zn%ZLUHc!7PL%)A z!8T=(rFUuE`-v1ijp76O40-gTGoBN}aU6ZmyckZB z{SB4j6uDVFeA85Q>A$Wl<>NEL3ja|aYP8kYe!-ff&+r3&sjeQcHUIAtc?JErh8w8< zlsX4_8+Q@s|J^4aBCh{6s7~36A*jIP^}mMGAFuy4l0FJ4Wk72z2N2ue#|j&V37CW_ zn1&e`_@r-x?ECDCp_1(XOj+18QCT!XdFj2+6&C0J#W56d?7)1-3-EaU-y-_s`F~63 zOY#4A{@*!ioa4Gu{o7cc<@6O;h1K|9&i~uZMxluRoB4lh#IqLbu>tk3KbrqnMSnd1 zZ!5hPe>nedhp@D^gI(kvWN$qc_K^pW;n(%}E$(k~$Z-M%K7SFtC)88OuGh6+pl7@G zd-SrUdBibxrTL4))DK3hAF#n`=XJ2T8D#05ud$!=*!RWnt3Snh1smM@rm`C=Jiqtb zk0b6OiB!Couk#)Diichr|Mz)y40?S=|HU@{Pfxm+Cg&WdpG4|`xY2^<&BlWXi}U}^ zkmnHFE-sLl@c#LKR~+}_kLLg7|IhRPuDRw025=iO|KBC=;~}2R|Ksz2VE$jb@p<^a zng2JaUjD%lRG|K?NAv%N(;v^j8A%_7Kb-%!`3?CSWBs;euy)VCc_vJtPr^W4TVMU4 z?_=f|I`04dQ(+o812w_`3SwkIaqn+ZwFJdc^Tt z8^|hbMJ*nW|Jy--JpOMNeINdK{NKc{hWOcrU-vcE`>7{Nwpnem$=%v!eKRe-oz{EW z|4{uCZS>9^iBTAfahQNf==!hP|6bDmhyU&PgJFLXrif=6W?&Yw^VnZh(sSF`U(BJ; zMUh@Wey{dHZ2Rn=hd91$zTXz0USHcHatZ3_O~=@AEOopbt@IQgud^NNEGvXRu2-(2 zuf`gzM_m7J16hTwsKpNK!X6C#);@`3ssG`xwdvCj#qdqq9?3rM;0W3OCC@j?pj{<~ ze|2a`lMQcbe?SwGjqa7r*H;cZ@kYxR3GepntwQ(e>h$K zzbOB!OSR*u^U^D|ziw1FCu9F$=VJW_UpAM(`V4Vyf2@C;6K5Rvvss-b_FY_X9NX|O zkyoN_udrDE82FWOGmcB^53v6?=xJrm0GZin{Pa3?YWiK=N6$j#GkW<8rSU0b@g^U~ zZ~bU$H~t%?adD&B7&av})Y>$D8`PkDMq;t`5Xd@h!u5LxhYEW7y0K^ENQ}Z*#I=1h z!pG5b?QH8|Ytqmsp|pSSZZ@`#t!-pK{a51)Y#g4eZQB?J1&q#Z4;&S|EIIx>|&g2*CAb{`S;S+Sr+HtTmN5uYI8>23Jb(p zf7tr}!WYq#>JxGO|LUI@3-34;`wrA8TEyS1e`2YyxUW!Y{r~5@*Npv;gsng(FYWW# z^u_Amj#r~F({o4f%iaUo;JzBw8A|K)EM(K)WZTi&E&n5yA8TB<7VEJARoIGJbT#_^ z5XYG2(EFBnQJ@!5n*V!V{ztme_b1M{H++`fd0749toD8J?7|-GL;WrG_uk-efF9Sp zX_Ai*(G&Q<_^~qU3+PDJs?{oAExP&vXc1nIKe06jf>$-86fY=658n4u7u7=|-eSqDsdj79@{^%X% z`J;fiMnGx(@ALBiW%i%!IIV1b)AL6s`o8qu_`fz~fboA*#5n^s&v`yy935uSD=`Oi zQTUIKh56*bziRvWO`M-CwAUoPQ*hkMjEpGkj2k3`TbX;(p zCwuk(#QlF0?PJ2G#oo(G?SHS13DwqrDE>3Ze8`S(APcE>5l&5NazJR;5{ zsz0H>olK(>$9-D~vLWO7Y!lB;@sQ1)O^fH!`anGHy-j%gUg<-+PWtlh{eADZ^nEP! zeITyI^oQkddc5>}SlQ*jit&GueL)=4t(i~Hy(#^#8;d|cgE*$}$^Hkg*`GmJg1kUp zLjB4y;R@N0I@gxgMG)6D$2U;g|KO~8hX0NGAM}0k&%;1G|8M?VxJ};0eLTdVCiw~l zd*BQqE3j+oUxbn5z%TBHQRJT2{(JcT=qX|M#lHyOoA(!C-$ngp>;5$C9jd?V*nbZP zM(a;|>CeLcd4CpuG<0M*1(_aR;OpPu!9hW4`mFm~kML#prJLu=XZLd*Dn3(YtG zEhNkSH8iQyHJ&%N^RoJ=^&jis`ETlT|4pCFheKQ!{inC=op#mOANRkdY*N^Yar6nO znK~uZZl4mWw@(V&UjFalZ>43&;NjYz;{3tE;qQix47>2%cKwgb{w#cdw*JSLMuqBs z7#ym@Q{jgT^%>#^3rB^$GygpNc%yHZJ)R`)DcG#6*tEDTy#18ns2t;|CHdA9ydWzJ1&ItyQe zwOEg=`^gC3Ku-*{{|{M(T5Lr<`(7t}2R*r6A6B=y3$g$ITk8K0tPO*>MojbW!C_Oo z{Q~*XTeQ8$@qN2olX+G9{#0x4(D&g0df3n$dXYx~MYgnoy=?4e|JJep+t}2DY%4q7 zs*OJNJipzw2^>KZDWuVf96m6=V6gZ{i{F12P(+(PiFO=)OJ4&z=yISWHT-ayMlgP!xQ_!_T6wifV(K|KdfKrKK&sE zCFLzN>@@yozxw}0_0yOyZ>j$yt}zkUpN{qaADt>Quk#xaB#bNgXS>*n6C39wk~Ydv7OV|BCl`-rPp_I0;iQ4Kpwcm6(I{ zS@Vd=`H1`9Eg-YEe4qFD5%kCFk1U}t#UHLeQX2or20j`8m@)6hd8<&De`NdO`Xi^ z@aO0mEezd9jpNb>+OCcLXFjQ|CLI5rHiSeo`yWr4|Ht3|ukDj0Q|$1+rMKuaJpBvW zVC?Bx_Wqo*In55wu*b7xZw>ob-$6eHFo*<(FpR&G#wz|7{vK=i2mB-c8B=-ef6QRE zMEZr&FPDC0OyBqKT1MIXV{HBLL+t-*_WxnmN1xo!{@-H`fpoSG%8&Rt^z3H;i@%jV zz~1lIpYikbUqtjBh-(8T$oLPrUlMl*{tbQwYtJbRze@gF{2Er-X}?acJk!73$rDA| zDQw=@S9QN|?}GflB>(S`{}I=eNnnURjEpk>-`TH4d-(LAwXW#Djo-z8z`r{GuSxz# z79%zCKgKRApOp_2@_mv_Avga|{rjJ!^Pllw@L!Sg9VJKA8}!)@^$UKV{@?M_^Z%Ov z=?dR7YtY&D2Z(b|9w7e^-B;wlC3ZDj`-%7Ar|17oT~dBw=9uyea}CO`qpn}OE=!J# z@?V_hzvy1ZF|pzLk#awi?xEM1e_uU6!A5?9WSrBVSl~a2VSK8{*?Y__;dUP#@*vzk$;W9#or;W z?Go2GTcyvk_h;&@>qh^3WMkg|(%JxXSb6)cVNLw%lsPJ&GS2WhKA*z>h<}dp_xV1_ z746xr3D@(^o5D@xn(yN0$Xn6fV7=0Let+~6yM13csT_`LgAHEzNq-VsZy3L5pI<`% zCEweMIzifZ2xnew4MVc!`SL4%h5oDPtx&F`zfd`^ZXL{5uG5FeVfE=qfqDUB#p;Qj z>IM5Ik#hdY3;NeL)GL^oSO1^;Nx!@JTtED%W4g_Acb;Fvuj4n6N=O6u(`PR`7QadV zcZh9<4AK{+CH~)w>r!w1Hu<~gxpY(b59EJBH@&~_+Pwb%Ec{+G%?HM&3t$0GV&*g%6P}U+0&^?*9n=qxd+w@F!Rs*%k)K zIDR`!uKwP(FiuWl7FqPR|D^se^gi93&T>bw?{?TB+Kf|BnFYs6RYy2&y z&dEQR!7S#GMh02r*8eY+|95-Wn4m}B*6a1p-$`Q?<7e5~WSsZ=_v9K<;{JjBN6cPQ z4m7Dd(WHGZxI9ZSSI|7<8B&q zo)`G<5PABU#_^x`<8Uke=kbg9CES63gI~d~Vl|ipL#}*1FZ>!gljQf4HW!ip8;EQB zW--^z#wUN%_dRZ(`)&J8q+0nUlm8w``}C>{`_NzG+K^k9tll-(ssC%;)0;O#=9B*t ze%t=Pi~oTCg#V2Hf~gJbA0XGyb5Nc>EN}0Z$B}Jh{}-wMOVt15_yzUo5-KTtyr7*ark-i7x7ED1HF4Z zfAm+n&T`k~UJ^B)zv~-z{r|>3zk*-Izs0ZN*YO*;A5WQotlqiN|5xn#ihO_P)&K5s zj2vH<|H<6{MajvAyzraS_;>jC=$hUPzfE>8Y=)l2&2Z{3w}#gVXZYa6KEQ!|-;QrN zK@JHIlhNld`d)PvuxAhRKQG=I-r$%wp=YN&{i$ufKXr4VIvR;Qb#w_kk{oi($O``; zj3LhDn_!nG`S|442e0&RpM3rWj+-H89XCg&g)?L}9l~3j=WTcg-i586|9i;eIEmhM z?Sl>H?sE+ooOTTuq7QdF=YDy_eh=ayJdEr92anLN`|mwUe+=1!$^^&6HuRkQ^SHPk z^kWEX?{_a`3Nu)Jt7|1!ekX)alb=QA-TC2* z)7M^nHGG9!`Ms;*Yh;}N_YLw}_zu2@arMU!$S3e4Tt&(^8rJ~aGUECxeY@CBzZuyT zb%S$VU;i-Y|6j$%#lSq9c9Ctn%*G{$FKiE=m);J?-G*o2cI?8laVMU#{+~8M&i>9* zMFi4*8 z{l#zaWR3PAJ9M9Yl8xFQ$8HKQp|`wP{rqa*k8QJiTwjH@12XjNZf&IY?ct^NdpTZ- zSE0w)M7O?-*U$$pI=_8hOMe}PFMBrP5?4Gcth_Kkyg~fxbGC&yk#E7<@D98S@4<1L zL<=6oLsI zF*&;_|C4Fq44JKw|2@Z1<0xYo#{}Yknd3hOC(*m0{!<^tyg49G#k`pyhcI03+kKl~R|pG%(0V&stgy<46TKW_U3nY4e3oOG`7WAisrygfifqd+WSb1`Lz24ET<1-@^~^1lB&cHT;OYiY=qw>%`Wu zos8}8o5|>F)l~0$YYbtB@NIYo;@Dnho4TFcCH!o}F}{`;7lzZ;6P&%`9eF?LZtrN? zyOD1*^gHeMT-=RQ`akX=@5PI-_JzE#k6d})_V5xij*Ue7|E2VoN z2PZo`1DwE=ar|kvbZ&gVhW&kz{f*pz_SoDb>-^6*_@A?t$EVE?=)30IpXxCGzmff~ z51?v;Q%tPwU3~ z7dGMu8-LXGjj{R1+5QvsWI6kPFZ&-4OKZ(qh>wtu;xRmq9`s`fV@P2J8LajehEJ0# z?>B~gZb$em{qy)DzJ%0y_qeble3?Fbfp5O>SLk0uR)0c9T-tGSWSqPI4e{T?ckn&* zo__iBZrks{y?7D!;U#z}UXEAdRd@|vi`U^s|E#?D2H~u+mp74b z!QhVN3ML{koV=uG;-@hrp*_ztsg_ke5N>zktxhzO`lJOjO+h@np}OpZ;kvszKB!L z-v7K|1b0ZQYZYY z-y>w7*fy2vGyN0#}oJwu42o$ZxJ_R2XgBVsM~L>KTxNB zSN~_!|5=PkYgAvtn6|>WHp7H-B#~;+KY`n%@eJIKU05@Yv+}8e@ND|(r;0Rq|9sxIZ{CyMh!=UHpI)M&rhemCyHy+|EZ$JTETFQU(?YiB&)ee{%?kg5_aOPaD z@ICwhPvA$mir$3v0g~2n$3Us^?@HswCC0yv9S_$V|1LECU1t2-wsB+O6K%$>JB;s> zlfqL;{sC!Y*yODExn^Vc)z$|vzduWkmz0Dp6Y?}}#tx+HpLG0f^w|rJy|gPlgMK@* z4aUojA*ac?1>^sFlzTNLzA5#fGO%Z3SJ-90XQSKp{)BOR*KwzC{D0U`x$CZUAJR{d zy~dpT=>4v3U{t(g5=-uH-hStIh37i%ZnVVthi}ab_t5Xfi?9z*VAKpTrC1)yjg}2e)f$WNFQxBzG+g#qR z@Gf!h!Eto?55!5bd$EN5UlLmA58@&8UvZt{2IpN@C7YmJTcM2odi2_Jk3Nv60-MIZ zkKpv%wai_hbW_!8E>dPn#&8GZP^Lash5FMN&s z2EK*wV7A`(d`z1|e)*noR@st48gueR%yT~w{{()7t4KA;;{>14{@yOU1GizSPyWY^{=XW}R^6IGuK#cTN#lR^ zo6yEeB1NC<^v|KY_&U6R-_twg?`QhXrFlDArmpn^oOb|%o?4!pymC^s_CG?l#Df15w=>I#y{>1fl#0TjKTwh20 zX2-l7uf(hH8oU;-!y7QAPM^jMW|3R}W>g)AEJoB*qw1+Ka(q+$PbP&^OX`1hS@a)# zlXTvKx8WVQ?mzf0`t|zrJ@n(qs!!wEx@mH5?@!kMOXxQ};5$4l|96YCUkkd$_a9|f z*D5#JR;P_c4hatn=jPdy587|-z5F`Jhw%s=#bbCJJ?O^}#*o75o2~aouIR@ZUvNG2 zPvf)rJW}p6i7(P;`#hV@?cq!GFC(kI&xlK7&hzP={)zuXvFk)`UA{`szua}Y$FDf% z$^OM(6aEIih3{cXTXGt?^=0Ji8|_QwQnp(8Q|kGbdH!TvQ)z;pEb;t{Jb&MG%L&ir zn_I#Uq}9dm;|a2Rlz%6FM8AqHDbL_C|ImwkKBDgjIfUUw>)2_-Z@2x_XXL2^Tf@!t zIF_)3ybaOUE{o`EmqyI1vv`JWw__K2&S?KPY4_8+e5-eo&&AysKI@$!fkEW#_{7%m zsC;~n?f2qEXr8DH`^cBzrFc1BiReR{8{bnty-GMUvNgPhJpDrD(^qc^ucN;KZ^B#f zHmv>L_V5n!UC4af+}sau5AUI`@I^XKo6-@ar+UU-atz5nfT zdJnRj%7!b-h->)Q`&!$-h40|=_#S?M zC(zqr{(tNrsL(&K*O)iEG&ldBul_LK^^s!sJ;rPs$HW5LcZvW1Jo}#AJGE(ky6{Y& z`RSNTvwgeSz}k&jvbTx-kA4hb5Q($w|K@A{|37jKSFvSM8v!?C2X4bNFxA2SPq6>H z+5c?&r~H3VKg7ra`yXQ%$3&9-pJxA)la3o_U*9gBU3fO`L~i{-@z15tYU|G6Zu&jQ zHn{dW_rNBfL(c!?(2oD(e{yg~xYvG}r}IA)-(Tgq;nZ{V<%l0D_uMKy$NioodMot5 zpugtYeuL6ZV5rsmL5uq4^r;YD;@Fqs<#;7th1cMz^G|-tzCWk`!?!T%TNoq94{3i? z`!)|7|05@bXFRi6{UTH3^nsuB+jur}^mM)FmggB^WUuFl6n(PLbKL1UqQBjB+qd_k z`=Af)_uepA9gv}8DoYI z(H}-u`#Q7pS{r*#{rQNvuA;*5DESzA$_vBeQKGw&?~3|yV3Qxux0NH};+WU4`f*VG zm>_#=)&B`~WVij&;>m>Y5INk)SC1Xtw0>><$BZ?6$E1~x0c|}tfl)uRRXKg{A zRtBe$^RH=B{>tN{g`Ph-UgG(aN#PVZX}^^>-WpauW)8qF#QFceV`;A#^Usj0-_%bt zxGj8|{#mTu62j-nFGl_K=KuR&!I$V?#;M=h7RHZk316Wn$rQdu{|3H=nX}rb&91M* z^^qCjxRzvBf`6aw-@*6r1N4aN#uM~7cfNoB_V6S6RSXLc2`9+Gs_kLRlzfSsu>&{k zqlyr26MhD6N6!E4m-X?`pN%{5T=c4g`;z)6(m(MpDA2dRm;I07GJf~HTf^PW8wcoq2? zycVy+8}KH)1#iPU@GiUuYv1MCj#I+h87dM_(-jn@E zDc6U{F~mA(Mn0cC?0XZQK^*(ZUQ|vT@EvUOpQ-cxBRBtr{OSMOp8D>6A>+7D?@v4X|GNIhqW15y@o#dxmi^nHonZeIeUf~UZ|W=3`5Izd_8VmPUj8ij z7X3T;9{Nk<&mz}XD1Rp9}B%>2KfDt}i#sc%R+ ztH0yBIB{P%^(@c(#QotV;$DiEeU_;O&+9GX9>haPE&6|ZvNAkOpWX2P_Vvo} z2>nrHi}xvi_l2~5=J1%fuBFQGIN1|vtqlF-5V{@LFP(vOrT#bR#`x-e%HMrq(0&Q@ z3ip}I-+yFZ7{DMB7{V}O{-1GN2A{@f@p*g^U&7RZ`}Mz==Wj0mY^{0y=KH73_s@`7 z;gNIqhf$1S91}<)g-M*4R=y)XyDv*`<;1@5HFEW?%J2=cMIZ0!ZN~YZRvEq}{2hD` zKfn_h@4G*Ih5Qk&V#|#C>~s$C+v&3h_JtYTOy7YlJ%hCSoL*9$j?tS4!9U>4a#CfnKIfS|L>RoF(F?kE7ecs?B6}?)Y{7M zD(SrjJ&l#&wd8}o$1ZCoy^fwZE6-ziMV)+^u08Yy@wxwaB-qdYWNb@#^@r;CPi+ft zp}!69z`O7s9LGtt;6XfuIIrkovR9eX=N|f9|Be5jlsT7h^pSbQK9AxtJdPgph@A_sj*XsT|+<%+&+86eP&(pt%anCt7|Iz)&e-C^~+?SE0 z&v-_&N$mpRuOZ{wvURSr+jY9emy4UhH}EZd2j9aF@C0(}=c&W`ky{@)q3&r?2eEJC zSolaA`?7<5Nsik#p+1ZA-r_vCN#QB+)9vcOgT~+Sl=c6G2aP`@*p@kavVhI$oLAA$ zwj7wx4_juHBjy?OU_1S0?7(g45;v|L@(lXz*o9{!rOr$q*dFer&sI3M`ZV*#5S}ZX zTR&HQnl5I4?^gewF?T_k-GmeR?(VivZvJ2MP2nE;z=pQTI=ky>{D9u8O}?aUvY?GZ z4zAo3?zPYO6@C5WKBStB)sino(sRkJi&&c%UM~DfWR9vAgy-tzKeCry-dE=Ny3PUD z8QWBe{ny%9!}Z(4s~qzhycVy+8}KH)1#iQY`ea&~Gl;&Xb7V|6Q{p^D>Yu{c|DyeQ zQTtPUG(jeXQ{<%my4d;ekj}f1d71oeUAmrg+J8&=;XUF8_Q)F-^22fZNel}Q?N$eq zgL{2<;vU39co2XbinSu z+8*S}f7Czv%Z1^D{^};gKAjygUcJBa3G@3up^yJ}ZVJsGz9rlye(hzy6rK^cKW9t0 zom_qU?O_-BY}|>pUUky9wJSeiJ;+GiOTGwk4n)6ePtZUFqwKbN_9uJ(#{ch<>^OFMT>t-O$L_;R@KU@S)u_Q?Oc__7u4T6zWk<3L z=Ze^kCG1EtYdn8MKfvgs{()uv1LTBovW^{zNym*}*%6LN=O~V$7O8UkuqW&2vpd<} zZ1H+}1F~$>jBwhx`5c+Ezek^Bf3uw%?U%E^JKYx>e4u(qXcl*h|HP2+@H+dHOdQw| zK7Qb7;jzMk(1DM&|6KTJeR+6vXL)FE|8?`#_J)s??hTKit*J76xUe#O=*<1$;i4CY z*2;?TL44q7DgSNjd|cbfSk8Au$(N-w#y_nvbav_| zpm!l!eVrxx2#VFs&*HI=hYV__IO z$kzfB>a6IaoFXUB`hL^;r}QE8S6gTBm^mr-O_;kfRBN5VorR&vIZi6?43*@@h%O3W z!lUwMM*shK$1TQK^223uSFnLp_buALw}egltbgLoto!%IUFQG&vi19l-RI8e-*8Kq zLtff5UAiUYlLhE8K3YibM7QJmr8AJeB@_u4W2joW#%3C9za=D4B5pTIu?J>x%EGeD5s6HoYt4{l>U?5 z)<~$f2J50ZeCPSn;h1B38kPSI`Jt9Ru%UmTGe6YP>oKfe7+O$glY=Yyp+Veu%}t?^ zY(liFo5?eXcJ@qxdLL(nTai(pW-*5});;4R;@i-U4s@a$edslIpX(n;>L0kQ&UH)z zL-gT`#`pW!_ZU0KzQ;rj`yMI!B&Hh7KRDnz_PdV5&Pz`p;vYa}EBODFsBfKT6iG~@ zr-uEl9!}E-4yl`Qj(#4)+BQQ-U=UCK|L_Id=W*(f)xYen1$s*`Ui+E6ut;CRGFEUA zm(Y7y{oke^7z5q6M|CngF);FCxrv67;8G3eZ346(MbUUtJI1aFMaE?Lelo@>u*idXXiaXr0M67rDtLoa~Sjf$B$}9$97t^c5|b4bBF%~kUb?r+Kct?)mLoMnswJiUu-*WzC zXQkV&{g3PYyCZu&ABN(1<4VEO`G*C83!FF36X%#c~-#z<28Uz%ga z7soMCrT-BrOk(BL#@`3;xaMDTm-Po;zcnmLcjajzER$;=v=4a^mv9;DSbN1CArjZ$ zx+1)RP2_*dH&CSftlts}=-1cZI`tfR_v`B4Mfo4s|GzoKw_}>$#f)~|Eap7>^s=&$ zuSNDkNf_}hk7A4;$2cbFx$iq&5+;+{hJC&t^iTVKFi1~esKNJx6W#JtLVh9(T~F@+ zL;tib?4%c=)ivjl#VEmUlwuFc&|9Z|LO%vDsEms1=H}MXC3EZhUu6GZR*sPqo60Tn z`uhG;%k2L;^>6*ZZvT%jZ4Y~$ryLciM9T4T{Cq!sHcuI7p9A!R$O>nQ)y+HE@8ltI zUD_g5WHn+xK@E8r-Rs&eN&N`v{BT70D2A$xTO+PrkU;!*ZtOeG{oiOMFC4Q^&r$3D zG~N_y=>tvdfBVGse=>vWWZ^i6Fwv?lLe|-5ypJ9BZFRu&J6*f{9_{i* z;U+}in=Ep6I@!D6`MSTHpMvs!u-5bKQ2)0%m;8L{yLqA6ac6KAt!P6#IxwXUpTX&7<@rrtuX2m{GPP z;{5ouZRe0tc1Qp9^Yjat#{zQx@_#c}pbUGl_6qhx zwf;YOxL+H206#tck+Du#xqT~8iT#Lc6~{bv0#lyhG;+`1JDBs_Z#;kch`j!P#y>oN z-$T;(kZXTAZspC|AKz#H$Mt`H=sTDGK^#IgYH%1wa1kNq!x%=U3|!v1k>|D69n z8UO#+A)QWiqYtS{`{cV0`Yd~XrdAsCB(hb;Mvt-ok2n_@^KShAUv3a?yKDRyS3}ji5n_&Uq!~>$waB~_YU{pZNG&5$U)bc&~GrbY`=N`9nfcR`i*(v zykjq59t&8+5~dowf6UCwf9JjbW$&NNjsM7dBhni^>;9YF>p}0o#`{N#K8d93SeDKT zQWZPGMe-8HD|dv;WE`uVbANHHHvac-UEE5{7oNk6XV7(YN4O$x1DnYEwD&K*yM9N= zr|0GmNVkAqh+)?^gao25$4+rYSo_OcLor!`-6+K#lp*uxt)ZN(z{;!HqU7oucZB_9 ztoxgAQaj+(`{n8Hvoo#taKN^MID{%xqXvg@1V?cUwWvcq8qkO)G~*0<`3Ci&UmxKB z2I&b5DIeneq!D%aD02T7ZB)M(DLYEk@8o2mISKZ=F@Nx~GJ{Q&>mR(NeSovh(TX;V zSL&xBlV$oB^#6CzGsgM5$(d>W6PQaV^P1HKoBGkOY;lwBzuNudR4{+vGsryCGk7Qe zfFExQtM|En$FG?Gmmo)x#5B@4hx53Ac`RTNOIY(ifQ&r8@=o7~aGd|QLSDorTt?UW zmatAp)cdcO|B+8Gz>W5s{=Y%xUGEsOkGgFBi7Ngn4_nWQD4QA*!~EIorX=1?ZCYh4?HEJx4T?V*CKM0d0MI^({u zU-$rq=))L90=fS8BR{Es8^s?LU+ezK;g0R$pkof93e~8=VI09xOkHvR+A1^T&#WKZ z^#AaGRPRr^{$tX(?k`wNk3N!hWIY;?t#H44-7gu(E3S`c?Dbw6ZSP9(GibQ>4W5YK z;P3GT5Ep$Pn{AK({)@hyEiZe3pP%^y!qK-g=RZ)lJ)E)6+E?|@kgaG#J37#bZk+mr zZ*NAy?k%{Vizg(sT9yiN-KZPqFXg7+0De$DU@?QRnFAk(F0(^q(7p z>dW^H%2#o0KaTN7-{QC~_OLo@=&*eD=EiWrar0QfB9^d>6-=+o9R~&mA*<P2<2qYIep6FE9e7c|M|R7N#BpS2FTE;^4~s#>a5=7M(^Nw=+8SI1~7;OhA@nE z_i?~+2XP2hc*^|0vg2WT<#?FEEas3#23d^kKORO89S>tQ$HRE_@i0+-JS0nxD_@U? z$)e+-H}Qb+=LbSR1};CqrhXtKiXI3^yIv8-*xNTV2VvWjTy`~s=tn^ziQndnf0x9 zOS2CNH2taig-l`^Y4mDq^r7Fg7`W*Ff}Tx+9K!Ho25 zcS~3lUcxd~a1j}EFjwDsJY0%#Q;p#=xsEH?z$Ws}`~FdYLacr2c-To6p%`)gZwa{@ zrPzb(^MA|ev+~)D`abg`M+1#`pD(qb6fvU{-u}OuL9ljc>lcot6!l~xcTe) zuIbTlXmC{i#mYZvtJt>sdv}C`tUMmq25X=a{m~3*F-SkU-albi>m^ob#I$o<>i*GO1TTAwBy3_D{?A zt@3>z+malz{YL$-ei^{v9(6;NI-*+rcSybI*f?%>jvPhTN?v%peOu@(-WEQwu3yqI z(};ES?0I$6vbu`QAiJ*qD_8$ls{hIHLiK--*|U;?BctvC)p4_w)?%|qwDVtj~;0> zepeU%cdd>9`}9P`h(H;2}SH-`_Fye*vET^s)J;_rt=X)mGW z(3`{Qw)$|Y{a9$mgWdJvebe<}*|rrtRQLAqfu{F{_gB<~vv}CKKGfG3F4}epj~r_V zAMt!Xyz=hQ*4SX}OY7+Buf1&BI<8;?AL|QYlkD;jt;fBOPUL;o{I=18&|{s-ZvS2T zWVD%EYTGTLlH8A#x3Xi&)$jg6I7l8s71nyUhibA0hj9db`nP+V^wA`o zr(L?8(w){v)9pNc*3@KwAGO~x)S?dcXh7zNw}k0~zCq7;)^*L0Y2i6C>)JAn`C(KV zBhnm~<`|iDjtOy-4dyQB|LI-a7W$UADId3mfl}py@+i(V$oUs49~zyf3C%czC;K;? z6`tB_{1P+T8S!6=b8LfjQoAM1J|IVqvHx+y|EWUT2&tXgAZYo7=l+Vq&?c>RtbKBO z=pZ|z{ssL4`ZiX?t-O$rka-E+;`@-mD3X}Qc-gj)j^P9BT=G0p^b6!XW-q!%q?cVI z;8FM+%deI?o=edk^TtR9<8cRFE20i{WaAtEy*raEkr_OVHCZ=JZ zIr~Jrv+FYZKiZ*H+HY+;Lf)Ur1L*E>-P5kSnH_IkwbN{jp(fXjLE94cEwJBs>yA)J z?nJ74M<^nTkxV%Eit9#+@NQ%-vBeji`-*FzcXi(!O2zL%8TO)Q-uj6srw`N`14aeC z62q0&phTkj7JaGyKT+WMqQAs-7P+q7t_wpL#(u{fz(E{B6{=B#!Bn_g7Mu;*vV zIpK7bx&T>>NMjUZrQVPCII+jGb)J;-P3oUGB8{UshA!nzE!o{&$cFHrzpwJC(|=zt zuK$v<2!nhE68-@=gyCiWEC)RIL&_7+zsvX55Rc!#HT0m7o?HJXsE5nc!=>_9rTj$> zVYtYe12t@bX4{)=$4}?KCTTR|49=nzZD>aarjD}jYqb%`*~V*g{?fu3;q0>ZUpE^d zp?`oJ*It^SCp*+LZTbfei!0a?I;GQ%J|vJTlt(d2&#kY^PEOLNk#)Tp`8kcazHVCF zIh@A@#Bs-YGXDE_fs8gqT=OyZ3q(75QQQ)i5&aM1x{xa$=Kudf@c)l%4_vhE5-wvM zSFnLiXnDQ%**4`{o4WoB%C~wph3$n%(RY$Xh-**J)c7{8XhRh=o!n4x%(e={}?~3{o_2zD*b<$M7=b+ z++zdTh#qaJI?ud`-c9c>a$R+7dEqn2waZ$y;i3(HczZZ2t`%))M+Z95jXoqWiX^7- zZ`n>qnKfWnkAm_1wI2Ub^T*5L|a1obq8S5CA=dO^~ z{rflQo5;JM|18?*7v;}QHu`1xQ`0>+$bOD2@GKvBm3p= zL-O|ldAVBtZt(t*Dwe-7wMY4YnF{B}96h%_k!{hJCpYf1Q`xoq+Pa0^WU2G+LC>Bo zp^PlYK&So*eSe8Qw*R#Li8lQcIAshW^Rkj~`i;i_ziS?^KCBA+$8w;OT>IG8u%A4D zgE)jLRHFumaRf(k47I33J=P{-`@JNryw~?39RC5-NH(DvXE3fkewJ)S8`=?lJCj%V zn$c%FeVa?_gKs`dpMdf!LGM-$jQUpD1HHz?`-+Vp?lFFd-2DDh*(gIUZWjSR9FIc)q=I%Cp{Yu!)mHh$?magO>Vx;l+l zN^cZNOrxiJSLjYkhaSho``dSgbM*5VzUlQx6#uc)rw!W62Vo%3_5$0}H(6v@&H1xpl(GH0F>+2BW*R!4G;yrP|;~?%{(kZn8)l9KEiwZ^5}SP^erj zVk76Xjq}*ZsCT{wG@=>t-+^byvuMSX{ijPj=RKY?e)|6ox%Jnp`ClF8Yei0YAIWm% zf2H!@^KUVy@bu^MLz}eL=0a#EJJ1=!@ArP6XAO|=`yPZ>pXFO6SJ(uJ7&Znt?pck} zlbFUSeE_NR&av$M3ePUMwt4nD{XDXxzJUw2opY^Z*HV6H5`O{nSU^v{`nP0zSfmfA zd;8h#OY~*Lf7%VP+Y{>EL2^ahllzY@3SYuytRuJn_h-}#^bKqx@6WYiP=K6&A3MJv z0~pkHNaU;IwI7DbLi_AQ5sFcQ-6+K#OqFSW%uThO zKvtpW92=LcLHD9(cR~G49u|)C|E~X6dC-1|Y2y!<^v|Jxga01}=?Pq4|Nn$GTGOWX z&AAYcNF%O!cKV(Aoaj3}%m3^A$5yry{(tNLuNiYbCaqf3ASivmoQ$^Y`(jqzXkD!2ZNcNzUWa_cWw z$^Y{H_#XLRn#of6ACpCFdp?JS(plw8xRWeGoP$wJmf+M|Zw9`{ZSG3GgeSZ=Kg=cci;g1AP%7lU2M+<_q3m0jT-b68b8Ki zdbCO7zv++Ak7D=;+q995iov`c;h4B+!`G5^NYU%b1|;b-=HSnwQMd^idKTB$3G6-0 z{;hS6qwL?9CYcZ(B8R7M4b6@@gR^Kw8`{xXEMO5!SjGx&_@5k6|6mSjWRS&(vVXMO_t)wB!vsB<@cp&={xH5#7%ocZ z5-wvMseJiR-FSsQdqlsHdU=DsiR>}$U2*Ap_Am10eM5Zk^T`7A?B`ER?nHN$di9We ztKKaVE=F!%U738W+(@A5GW!^7e`yY;_D_j@O40H{cJ1$M4SVQi*o$(+b-ydfRrBU6 z$^AHhgBWkscTXm#wf`HphAMhBY7p1Fn#J{XNHWUQXan~ikpHV)CkBqXu1422s(eB8 zZ;IpphaGp#`X|3^LlLeIb0^GAlB&G-Bl)X%u_|BrglztQt2u;x9W`l zCt-VT{U7(%@-ol!3){jC{~z}~h@Ab~t)61*50KHPJ<+L-rw=c053SPZLjt2nVj5|j zLvH=wdiH;#etL4QmVJGmeLc^FDQdrbX%MEirjjBK^~7wMzw-&*Ytab31wBrl;y{AF?--F5Cu`f**|E5aKX z+UdTK8|QGI-ahs}`d#0^sQ7vJzgPalFiu?1)qz9|v#{hfsxT)Zj3V z;3$rv7MZtOe>dD0Rz72#`nQagz5gaV`4$ns$u)go&A(;))3qt=zxp@78d`p*GMqNX z)*!t0N%Kace6ezXY{nU!MJw8H>c0CzJK2FwbYs-U9A^tD&(qkEot z!LRmB3$OlB27%G(wmr+0UlAJ?`m z{K@pwa^qy5xKa^1^Q}K{={{?rRfaPA?nOB&FnaD6!icf6+&Ed|FNEZxaWPcdwjU>V zmxe#wQyLD?52EG#o^ZP9dEr#?p3sa3^UA{eHp;>w+p6(U-rn$mGcO45Z+(6^Te3Gi ze4sphsJJ||eA{@jKG{cVD#AyOyfA$D$_qjp{n5_*!bcBOh8o8nMpL8u4jMfQtbc&nQtKa7 zTK{0vGrzKr|HHm8(rEpI{nkIgI3|!hWc>sCO_tvux|;3_P10$`8JtB=^L?Sa?Y_`T zAE5V7-xu2G?HE3DpSH<;#!l`FgE;j${ce{lLx*jhNL{K7-DDq<^qE9um`z$!KzJ0H zj>?eTt?kfV8PaXme<-y6K#BDSc3Xd-()t6X_ARsi06AP!`IPp3((%(s<9tkOitR7I zPHT_M6dQlToVH3D8Rv-S8Ijg#x%DS7o?jUzoGaPM_D5WQudDCAa6wwR`F+Zjd3sO6 z{bPYXuu$-(r+tH*)roHf@|wQ!2dHyD#h^%cB0v`$BJ-^?z%Py&kgVZLN2_b1ul`)Vy1DS0&stX9HnM08 zcG=h~Ic~p+26a19^vNUY-wO3FpZfmxpRE5o&Hl&G68nGgrcmh|`%&dO4v+_ND27pu zDQ$>pZHXD<3A30p=8(pX{|_YC|N0fiFy74m?=b$Los!aCnXD71El?w!!#IMYNV%q@ zYd=PxRkzN#)>?WU;+lq;8uwV`UbQvq#dT?qG?0ym{|;;-o6+sqer=C9um6njSqznE ze_YYd*sYyW7G|G4r0uS#WesdrnZo+euz^W^@OHsN-3AoKRTFxA9A=&b&aX8j-S z`ajO-|4^S;laRf_{y(h!q1+t9I3|$1#J^~Pe-U#1hu!|q^+NsscU{XD-O}zu{0CV4 z_t)%h@Bgjp*?3NRRv&s?+dGY}746%&FQ$=3wB6&szs}JI8A0-Uy&lwVzd0t_dciaLNv4mx;;3DGMt5Xf`ot)85%k^t5 z$z$i$!5H;yMlinLGdSqp$%z{GPNsw}N#iosaRnP#GY@~0%v*53#@JT|^4@JS zm-@OuesazNafKK^rhP`9`i$pSqW_g%f@G0vD$p)K%eVap0GZ`$*Ov1WU(|lHUn%yW z3~T>0guP^W)L$Wg>tkP;(9YHWvT9ywh4@PB#{nF~A#`2TSMe$1&h2@jO1K(5t$Cq_ zJdA;6-xv}#%8x4F)&r2H3QNNj7l$R^@|r-ul?r zZ0gY~?CZ@qm!ExonLXUbp6*~@pRukS8-BW(eSS9D=e|$REsYF4i>40sAsU@4wy~SY zW}LxUw8XI~?ZsAl8`{x<&WLw3-pP7Ov1Yu!+1cdA`1pKHp3~IZ*1|R6aHT=i8?Mh1iKA6r%*YF?B)y$IOyE zAV2@i`WMn1RhEpA$A815oNsbNogj@pD8qQGxx3``^=ZrL6{y6FKDp>aGPkMT zA~T3{*?QKug#ET3K=e63NOoP=5)P477)q$@)^L=547I33JsQx6CbUE!dt+-DDpU7)26s z-orGR#yOnF%3xl&K*s*{6MKD=mGXNDd*+Ai^tbK^^KqZ&`HtfH0Tpb7Tlo5a+x5IY zKYUdm==J}XF4}L_Gm341CHgY5^~xor&wG|wiD^mqBDwlId|}ATSjQD?U=w+N#oj~# z3b7Mw-@G;S9M!i%FGhE*dD>)Gf?p3=ilG&G9dXT%1p4=T{@7z%8G75*zv%B%PhoKS zTEFxVeHeS~^W^%cT(|<2$h=egE1~{Rsw4Fq%#w4$X?^V(WHHjo{>K=`F@ZQQF4e96 zcitoJE%pJ#^aH#h&j);>qw|5-LfdLF6A^#4^*TZNsR(}H1jy<`djqpM9@4ff5@HtGKw{4z{P0lWB7tqtj_%m0u z6FPn}K5vYE4C9zU{P#x+lNdj!Z6KWsn8yNA;*waT&o)aNOY~)A=^3QkoC_=Bx(fNs zkeARyzf7*9n;!r9Gk`0?8_2CIBs{pF4T4ywuIPt|>+G*0_M30oZ}l%j$orr2915`Z zLi1;%{GmFNEQ=|3+?I{0?Kx#;%9Y?h3o@a|B0m47I33JsL38XZ#y8 z*7KkB-}AXT&UJDzth;cHbmE^dCmvChbsPmHMXl4TJ7IW@3-&3pqPCWME`Vd=I>ryU*WjA^0|35?>#{M!cYJj`he zrjcvsEz1AR?*FX&C&xS7KbaIxk(1hDcU=RrmVHlSk zvyLmsyxuqfxrzKm`2tgC$5RWPRpCf$QN0RTr>u+{1_v1|M#OOl@qB&`9G~} z>GR&$Dksz0=-pdGp)})vIa3L9%;=}Su6`qD%A{W^{YvSR8DxvtAD8$s+urhK^{=*K z3B6~x~y+Yx@v^gS4^ac|-h7_8eG%EawOIVwZ7zr=ss4YVpVlFE(+Wyl3($vM7gj=6w&EFjmPFzS6_QFw|E-!x`0 z%chi;v!E{w6wf5!jMK3(d$CFv|9`k}3mJyqBI|1Q$6ufLP4|KIDI zOh3v0kABHMml4+=NjzU*5^0Sd7bMJPrIZv6kKP5s}kU6PLNk|*{5Zn%GS{n(Pa9uuBx64(EK)Rkv$ zmvl<82W99fcK@EsUiv_RYpXP-ORvCip=(0|gQyg@9|v#{Pqse}37_%}#&ru$e>;Sh zSKJz^#ns?2j-aqvH^{lx*-2!W=Z~3 zpU!Q_pUAY!pBOptB>%j_>VM(nA^Dq}wBPsvZ4Bu&;|$Isbx8e>R{HE2X*X>RZS;0z zg)_owa;|l2=n&VH$A_2fMo)qJArmo7??10xn|BSuNel@O3lA!P5@c?D?^u7!zo&w} zMEk6KQr;m~jlG^HFJK;P&ktdNT*MNV(eicv1wYh|r(eV+T*lfLwug0cWyJSL#((>6 zkekT+Yu^+KP>7uNWr$2C9G`ry*qBfR=7?TtOkniAK`UMdrpvp>FI{kb<= z2hjNb%3pqIYieCvZ^PJ*?Xyer|AKGen|#oPW1BP+%Lc@M)%31gKMnmDz~ElCEru|R zbFO!<<0`Qq2XGLFP=%>W{QEJ(7M^Y97td}^vyC&zCjB3vn*aG>{^#Vl{U$c}pCd(| zT;V_8P#CJEQ-i}ef}T#psR`3rX$UwEc|v1Rwa;QlYVfAM|x8NaL!AV-n&{UoOm*ZItiOV@he!sn2o zXAgV#(FUdW^5O5}*WbTe{>LB^h--Wf*y2e5XEhe8P2ivTLXQH);1Co>g@x ze1GM>6Cp@Ms#H({qEbbrN);<;RGvyJ(=shnsnV9IOv|)POIt=`r5d>d2@oMbfB*pk zL>DvgXXN;Ud^cC0>qPVaMF@B7F1`u)~nuf6x$ z>wMaI@(c#Y2@CaU_3hg;<`+-n+_5)$#$#Len#)Sszi+2)~(hp8GS?F7q#V;W{(2P&(SOXhhiBJtAD- z*6%$3jmIwzo3btrJC$?2PuRc0zGs{B0)OdSn?K&J?AoC`>`7K1uTBZQ&;KEw!YvhP z=vVgiueB!N+jz^kgR&il1F@O*%!+6Ndse{Z6?cceNvP5nPBCCqT2nV5yy*gxC;Gt8k^cT^a&MvFOe0B@45R0%FOOT1Bh~g?6QF)og9L*W!AeWwpvSaH1Q|^K2U%!s4co$I}Z@gkrq^|D8PTf3y|JSLmR3qDDTeRn{IR>&_T| zQ!Ynkb0fXUxJ=!AXDkS(6y?Zi<5tX1rsrU8ZSnnTg+XsXXNIuRj_6?S67o1or+BaP#uW$bNnt*T!T-K2H6GNb%pl7ys4p?NKSzCEIZ?ehSsR&*{=Iq1 zHK);y9{gqduaCL8Q2k%5jxJIEqiu<{B|6ZFnltKN96qnESNGO;tN$C+|F!CWbexEX zGr~EG^SFQkB%PKAerGO>jK=U%$=;i-k03{4@I2wMZ^c_@yEjI$8-vb~!d$9-u*9>> zc=w(;FTNE|=QaTo(Wu^Uz$E&PS0}w1FB|!0yu9em_`#&t;uYzy$Lo_`j5jQNDc`%Xbt63dc|Vj7FqOc46du5r~I7b-4?7Pr$pY|EqnB^Wz-;ST6w@!XLeguQ-^D3puw726ODR0Ny zN4*_y!`{Mo;-~rRMi2V1zx|zfY0EqDGxUnH@5B${Ed3m+z0;v%@5HN4y%Vp*d3JT@ z-i}`&2hb=!lm1=%4h{4pGnEUorC;VW4E`66xoxgIOqO}J@~PrKL;TMY|1-otd1#6F z$4J+V!Wg7u0w!V-CZl9n%*8y+ z#{x7jl>fx(U^`nJw=qZU>cMmV)x|a4-d{L}g;z%o&VQ6^9~Hej{}G*uv`{#Uuoz3w zI8hsFinRswXzt?(KTGLZ=v-_pf8o1xA8kvmHDI^byXKO4*w24H8J$5qxc-1!GzZcl zO%}30G{JlM=9L-VbEibuc)#v4N6uG7tB`8HXs!)ptM1B9}qw>~i zd5nzu{%!1{zJJF#c~d@%#sH$T_oF>}(f;&k44?_o7(il{J$P6yZ$@JQ9nU*^n`}gj z@CX0fXgn=C10)&?XcKOG6gE8?3)rh1YUl4Lj^Q|>F@UJ=-$k!DrtH8;`YBY?qcMQ! zjL~Qe;555#^k9bc-EaP`kA4Pc(QDk~9NDurDV!%SU;ru4%L7O(RzGK|kFty(SjV{X z72^RJ)?u?|IO6To~@rvwxONgL3S>+K48B7 z?*;u|{n@%w{r`6TUozTf-&CkSBdmV!m`mm%9|b5x5jw1pY2`o{{I)X+oJOymk2Wx zOVR&x`CWaO72PK?>;HGWZ2bRa|F2-~9cw&_jQT_Qk=YrH(fPv#^g9k z^!LsmE~S^F^R#C>=^6QHL-fzrIqa)Yiw5ksR+L$tN1o6#{x z{~tAd>i@Iq{{i*CHqhYs$T9W*QT0E6WxhqZZ*nkIKOL20^viLGUX7ebe6OdBJ(Owh zsDq}u=QLzsMs$CET5!ZQts~zp)7z)HM%y-O-_|JqYiDTx&y2 zF$=R1{kL}x`QHCE=F;b3J_i5Kp*(0SmiKbyL4G^QXpip#?h8>WUoRr}o^mEFxdeyk z)!py5DXPeWXXVF}*37)-ToLYNQ_Mf&ASzH##X|A!p#Tvj;YjjGi0YswR7*DG7JH~jf@QTs# zu=S9a+t0t#|JT1^|Gl!clzn;R);2a&k+o<*-39FwvK>cp4AI#R(HXBD&zln$_eYjm zPsF|pgY|EQ{D(GvdcHj*oMhiS)P5JT|3nC<$!_#S_V#=BMf$%ag)`*f`is}bhUou) zXPNgZ+s={aaRCF^FHM!6cAnI4r2nJR)&=Jy(^D~c{-1ZQLM1;X{72`KWIN+Nfi!+b zVid-pEOor|X`F$dK0Z{8a^`)Sv+vFCA3`;1Mvk|>Y<7D7QYK#rgnr&^t?9V~>8i`%J(@OhV&HVU*f~PH&B_J@5Q|`c!nf zPscQ2WC)ADY3wpE12ZuTJ&%kJv&r82v0)B57xT~`(!+dm0TyBr_D&ra7L!YmiKW&@Jn*5*Qo22_Td_k z#W5U|*DK`R%Jb^Yfp_}`)y&7ab>SqU_U|e3G`i7)!SO=rD_S4Z$9x87v8vu$qUVzb z$N$bVU%&wRU$hSX1?4$X(fgJ)@MM(!N0Os32I-i9x&`L;$w`=uDR^)H&s2KU=RbmJ z^bBxXs@6CVLw5XqPI92?hevJ4@5kF`WSIy+$|J4?Y--Y6Lf%wIH`zv$5I3&yv z?o7%jiqWi&Y^hTJm74!fRlko> zzhm(HpA`N7lj=VlMjig={{u#60+tA;6y>NwW2*jdlI!TLqlAqHdJ8(2s#}++V^h?% zXlK{6CpjD?d%vhXk|r*uN^cqFZrL427jhoaXZ(gd@rpdU!8jqi{%y|x&mR(UhN@eG ze%o4W8rbj8(|*Vv5>C@A3bhT;P47Xqv4ca#4ytmsAJE6H?#xBu40#rf^mF8SG|=B0 z|GU6Efd6IuPk+1|gZmUS#Q#L`KUw_G68|%_ZF=Mdq#_L?F$!alj=}N2S@Qop=~KPa zwp98=hjcvn|G#7ZFXMmwO%TRJOu}Ak%O;bhn)I6@{iaL55or|svJLZlt zM`u0Dym4XMOm*ucAxsl~@2Bnw8RXvDSa|l7dZ}SZ*pnv>B;6TmQt#2WoE{G6Pd9h) zp-`W6oxO$AL&vSdL;3MLLfO$fLWO;52YWth{Ngrc)JH<~g?qxGq`RF{Y3wozckw0u zuNL(ShlPF4S3Hn=aoArp%y+jg|9<~LOMNN60y$Iu5YHWTd008^imEzENL zGZQZjYtk+YYl|-n`DtUrx-sL!dUfHssLTrSuyOI|uxZv+VRP11p)kdN(B=&ZTW1Xk zvt2s}+h<-Bc1*VKYmWIq?>v`XkMwAhX2?#8_Lur*(%?My^RWQ^=A0Hr{_Ou-L`LU- zFD92D6H5{O`!kEoMlSNuSgieD6bt$E);#^Ec6|hTAqLMco1<)>r!7~Y{XZ!!6tOQx z2}-elzI_Dqj0?~!W|5)o?6(4DvT{z^|JeSYY%O)Y^FKxZ2J#fUx7+gO|<{OJ*vr?1@<2-w*SCb zLmlcDS@(~|bo&oR*?$l{(^TUVXiaf{_U-4Ct?0zw1($@=?vwL_q|iv+P}>0`QDH0Z2xonU!8ZjPy0l@SASOf zw_E#H+n|Yi)PKwn&J4`NEHvgyKZW|#^k~gpboTum`dkeD|89~oI7EAZI!+{qz4F*R zZu7DKsQv)C5Tz&8Jp;-Ob@U?U#W=LsxV`!~+WS*QFB48V4o=hlMrXoKrlNA2w1yY;h$pNEdilS4k4QyB||rTpOZL+)7Wn2PatX>ZK2cADN=q7I*J9)Lazo#ome?Apn;2Iu~;n}hyWQo>wv9_C{K79x?MJreaV z(?eLq-18%Q3CUjPF)ShX3L}#o9RIt|Inng}r_EoSbk-ZaLL49LGR{xWMKpFFjp0|3 z(O5+uySjG$1hN2)>hwag2o3ZjbF~ZRn~!2H!F%HZ(HZp7e=y3HXrJOBD)7Jj|EAJj zSl5-K3bklJ3)<1V$ov|Pq#B<`Te`9#MY)h>{Ew{B|9}7gH~hVSJ|DeFxo}h%$8a2b z^)0%{(kc2MI7vT+(}?~J5d9;dQoAEMxBI>GdHY9e|Be#J@_IBD-_36yR=#n!a?{!Z z`dOUAdGvl`NSGi^_I%CWTju}U_}>6ONxxI3BNb^FiBTAX=1h4wOFWaUUGlVYyuDi< zKd$|C)c8-s|6%;kcWIif?6J2mT{zLde=e(z8%KM!Ra{J)w%9N@TOEE4e4+l6h#Xx+mn51!$c?+a+y9Qkr|o}7J-q>q{4_1OcQEbj?YUQ=>J;oh>w4jK=x>&u zi}vMvF68{qTrB%T^lDV13U%Fghhyvq*B6+tIZki1FTUaY-Jy%#YVZ7!T6@Xqr_edi zv(0f2e%eqr$NOXO{TB!qRe1mZUwe|mY1eh52Yon$vp9!lbzuvlI;IuxxBuy#s2TW+ z_P_eFzC~SxMtYOBXI)=LI4_(F7(h~==kKvc-roHbdh}1;Bh&oLj-H0jEMtFYU#k5$ z(zz1s_LgLXQREoxFU<(){1y==^rx#tagU&##2z)Z}-Y|O!2MCbp^Bi}p!XFh!a7UKWw`9IP5 zgNt0h7)y|erHJ^#6^icaWU}`v1bJRmUC1e?R|L z{X0kfi|G8DF5#TSDV#>*B=xU4uAAPvME$#1`-9$xPJQtXb#wbX_3r}h4|dV{H)qLn z*w62I@&Za1zT0-GP)`mpC;f-?H_CgR)Xs?d+36Qa)Z$M45v9Kr?x{#a&hy5b$dw80 zmaFaWrjJ27CSW2aVKRCjOAb@W=uEt+l%p>QcOrQV#^%v#F zWaP~%9HLhvTED-5zlB(Y#aMz&EJgET`CmWbh(2U1+UV_R%Kj*H&xnzg#&$=5yP> z$b7qTs2u&*z4_Xc>{h;EJSEz%an4wQ_HdQ9a<{n+7Q@ce(}*x>p9 zXT;3~afA+fCu+oDEe<0(E3tm0_rZJT|DPA$1q>kR_u3gqMH>2l?hGh0>i>@-d-pjT zVfd&phMtZIh~_>gl9MnQQ}EvZzoybF8oV2((KGPgxy4nb-WBh!f1jnFF;Bl@k$wdZ z73pW-oadV1x|x`T*_ea5n1^P4kd^`c46;=}WpMq!{&EKf*Z-fdxtxoYF7p z)=wduy4W3chJVz4Yfye+zA&S`0S)>%9r|RglZB0(uf)RwZk=P;k2L>1(Y+?wBfxI2 zd#zk8pE{!=T06gp-C~rM>bo!UURc7Mi9>U}m%dQtV(+PsUzg|1HTGG^MlKq&-Jg9P zy|vK&^Dhbc^a6BVaDVU7?w)NZWLJb@l%RLGu@kaqwDA+NztWjTWGx!df_5CmF_b0g z*WvyDE6}TO2+>}LM7Oy4bNV~Ze;0D@mcO0rc#7VfF8;;Ok%{8}jQoEgYS(k8ccNyN z_@5*GQHT1e-c7iTWRvUm=B0(x!dYo_z%vqt1FiGACRx^OAd{5qBsmJ%(r(!V z`CGn=&cJ$q{PLXs1FA8GzjRE%MD%~gc~j(MOhIo>2vf0n%-tCLFX2q> z6!z`L0D9Q<;SA2=9M0nc`pZ(n0NHbUN=P~*{{7Q2g-k^nMq+Qe|L;wHw+#`+&nR~L zPkDZAjxqF#bDsZi26g&T@!IYAdptk;|Ly(*-{O#PtI3X+js3ZHr8$aKS4*4p z$(VwvXwDK(zTuHH-+*ir$L*=Eo96rh)MyXYW{P|M>d5+ezQIg!@B229Wd-s_ku+K& zf1vWDwAn3||m<-ak`9HbAfU(T?mm!5@cWl5CxtH@~ma+x|XI)nURyZ0w6 ztGvH_aY+7%=3hH*9vk&f61m7jJ_@nwYt{#hO%KiTR7<}07uhOrwb9?7-(70_X^wUb z-ar5Goc0$6|G(a64gW8dCF;FwLIqMD`dPC?|XGv`4`DfU3w&TB;=* zFu2}N+~nLhRvXV8F1vOdMdw2Afc8_K4e#v-I>zld_WFl-0ojF8i)WlC<~O1dI!>$x=S zUNS!HS~?~aUlou;tvyP?&qOy=#-L1uzD_*%;cn z4}}%^H-?(#pb`}dRE_Qr*v94T-yiDdBZ|w zQnI#$@-F+LaKbsEC-fP9#BGx>Um$z&A4nBOcIiiAC+QQUr|pI2)8<|qYb#LR)kfc2 zd8bZkR;IRaJHotG81>5AhA!nDnv}bP=dtuD4~vcclDQB5FusGoRd4y%V$!g%mp<0C z)&X*D-|ﶪO#T=rhnBo3J-sSv_85Y(I521wlZT!SD{T$+=LT}SlDd;z=!D@ zFPB~)v4=6l!bh1ujplR1!rzc(;^_-y-lU7e-;%rCqwQ98{STA_>XHTQPT0rMd~8@a za=-RrYDj#A-H$H{iQnKq@onZmk&lwYhNy2a8W(U2ZuQR7$veouA{&%*|3Ee^xiO}y zP1G8T_zqc@HYv7&R2GH3q~U;s?a+yqF*n8jnQWbRdF+~@+CQG*Ub0bme@jkN6q=Kd zhBXsECT}9I#8#hAq$CqGL*#eN?7xM%+o*_w7u>>=TvxcrXTc;;wL zSciNT{X^4{or%OE{ku%_uS=4`itY)qoJB)IuDSn}SjBF2iFG_xN#Pmu(QERDgtf*W z@=KG#r-iki+|Xk!=-DA*qaRId@`9WBDV%eWJeCx;>UVEDsU4=?-LC%Kq5R#s$p77n zE|2ZvXE*j_ULGrHxhOn4%U+z^$+3MCj5|&?4$1yNgZjB-WUPG7<*~v46J6sy+NQ?3 z?w%U+_=$tM?vIbJo7l3~k4w{-Z~hPjzl& zmo+BbD#@y2DdAc6ds9-v9``7jW?eygN;ts0k1QSKyc^H3!}a%&bFr&CCG4J(Y@Xct zU8hpQKeEfiKF@sPwIT5<@34oy(YHH?^VsB@r}+M<*e(5UWp0)i|B>4|Y)2`oBDkHi zlzr%>`Up6Yd|Rxm=GNGWtH~G0OYt>4d~2+0ZKO|$bzMFscH-xeyd!qv+gHcB{^k1E ziCgK{u^)bStm|v{#!igBH`aA~Br{@N-}^}H#Qk^Er^Qa(`H@&x8S`rLjoV@;o^qf2 z$Q#I^_jE*yEArT+r(JckEX{?jJ+<_b?J4n6FGOe{x0t9+t^p~^M?Ds zaA&ORu{&da2buUU@ncDRVuWk< zb-VxB6@55XRXRNU6LZC}YhrQkKj3~X`7fklvBV_$eC$mc9oxh1O5vts_x#~u*PP+j zhYSzbGw+x;A$FLoM-y6cBMjLkp2MceABz1O`7Pi2AF)1Vcv!bYIlXXrc#3(=%&TKx z_WXHMZ;Aaq`43pe{!#LA80t#=Ngl{{-^2n0`QJXG#iv1s{%d9Mubq`{n6ST|*QZ2i>fu`J;ioEZ_8M)y169mGdoq(5(MP@nY|lRVcH&*k0o zqi6AsC&X((HjeIL+wm^31E@!MhTNEF7Z zzDvT17lp4ZZ*Ga|dgbdr`7&`$tm`RxZnJ!zJSo=ovOMz@{*0|OA5|xux+JW|)>{9# z^gYU>XY+1Vkz}qV8?bxPU9pqosmMNxhaY%nanZuu9@+Kz2Qzu>&wBG2&x#W#Zi}6c z>?YD*ye-x|!Q3WuPh=-fqi-9tpNthJ_7_hH6_5^-tZQVbfye)paSMFnws)lIz;BDcUpEOk@lV z+qLy}xPGVWi(P+0zB+NWdST(PP@=8$EP0|$y-=I%ea-)p&BhbTvXjGQ*Qx7@oDn`> zd(m@Ll_!TQ*i}y*YJK6*P$(>$%--R)x1>iADz$aU%&&Yp(ZIc)Mhzf zIa59D`uatw)=sLY`EPQ4GuhHJF?NJqE80#@jJ0Q^hK{AF_OYgh6}P5@k|&0QSG`NO zx^*AD^ixB^WtP{B#TsRfA^F60Sp7h9(Ctjz3{A?!?a<{){09yXH?>&(ml7*9s6(KoXLgt!JDZSb^rmy}~axm1F zc*XOrkLsJr+Fu%xRTIqNOd1k?%Pw8L@-fsH&-pz09jwE1u6r3Nt{n%>ip2eR2>*aK z@3y@Arr3(Un_`Ip`QXA$vBB@|+voHQ(+`sF#rGPlMjqDXriLunuI-Tz@>9dcmYZUm zW?UcJP;^tQ;Mm1si*rA=W?moLe9S+6PhJ<>&b;IFjj?TH(b6GdgKPf_I|nWfrZ}8G zpp1iQofZ8<`q@!2Bbez`tPmFP{!SCx40O@dAE_7on-2 zcm=QF4ZH>2g+wy$Ru6vST4%Y4qdn&zjFp^zF!t=32V;BBXNG-eGo5|=c-UX~csS7h zxb?x0hw`&eg@Z?*3KbKwtog|@Zl1eqY%w_t&%R+APvam95Sy+|5EUeC57V`3!g=fe$XP1Sw=a+^2 zfn{OcnPp);HWV!j1=v`=%zwa_h0QI?LLs)$w_+Q&qB&28?PHz}JFqkD=}>(B>9A|y z>9CvJ!(1}zY3EJI3*xmut|n_- zS3B|PaCq|5p^mK25dUQ3yzhjjQ<{}G44tPlWs1i` zcE;mj`K-soikXjxoEeXY+}V$Zl@}fhtFj)~|FOrN`I&i-`w#ke!&?6Gi@zJz)jGE) z{kvg9!RKNHlO7KnvmXzex*iXkPd^?C&pa+4JRY{5oEO_x@^~m(`1#oObB~7|vp*l( zdGhg4yx_ZGSN7*(yH7o?|C1F;@;@JYcIrd1y{V6feG?uJrIV$Xf5wmT?-; zXd|%z3$X~<$VDxdzxCa);>GWVcKR`_{PvS!6}sr%cp0zZ>RXkOxEZ(NPTYr$_dOLp zLw*AV+Ht=l-@sdVYKruSil09f%3gdbl)vy)*vz~Qdr*dFYn}=k6}C zC*Jxi{fRaY5!tnyZ)~}?`r+oyuKmfnX~%8d+M}~5lEOab(o_1s-AUm9S$5VO z#d-Dffc3FtC36+~L)@#$8vbh=jIX-Bj;v>Hm}`7>{=4(rgY)B=#%HsPeUgLcA8_yB z-brq~O?==b^y5{$j&~5hJtVHcZMX|hVi{Ip4K`sbTG5GL;Wy~T@9`$aPt^~?_4p*# zU_GkQi67$kxcCm^ICuzO$9J#{tFQ*^aR3$gA%2Em;8*xP`td4W$2*9BR9_2CcnSS@ z9lMH#himUNrh?m$sIvdwfWUGcqZMQhbM7f?Z@RUO$<=4$A@6)VJTnjvYsj_C`Rvzm zUr%n}zra}hMr^`n|86Oqdx`I_ZA@-6&svmuiSK`j@9&&0=3@4{xbG(SR5?ei!P@+G zbCKje=F(G_`2Lsp{?@&oH9vaZ{OEvrT(Xk6iv1z()npC-wGHMsU0+AmGdIjNzd7Ih zCfU5${Ai~6(Jb?$WE*ok`ws4%WZRd#2XwP9k4qy*T@GSOX z=N#=p?80trLlL%P2R4t?p2QYJ?a2)@<+s`L8@Y*j?L_%;viwM{&yZhJ4!=P! zevg;%DqhE%7^1x}0$1QFT#M^*6K+F7d6C^<{7?C@g3MvgJ!Rc}w{`dA>a+TK=Z*gj zBuW2C(!aH~?ALK$Pj2A9pdl%2bp0lBGqWl4aF?*}x=tN5IV8S`YbUBpo^YOqzIZMB z<>nVUwrPx6UzF`Eh2_7yD6CkW9CChsRmk0ZrL#?pSwGUv@zjtpzV z4I#h&rm(L0#<0F^l7CY?7JB#i_g1s8^*OVokPyNo#()id+I ze4D=RW_Hrez9(GsHF}FWkzS-vTAdRM*L%(=%)#gLOm{OE<*OSv{P}s$sqW*!x}A`daefd+j>LjXwCk2IiJ~x1xpSjlbKM@n?=orn94&L@r^x<^R)qkDz`vU*;T}k|<3L|i#cnDv$~^BctvJ&K?H9|+%d%P;lZ!s_@2w^D8ovTq+Us7K$icQ|tavvyTvj`Cyw z3I4^!@(tuRWADkqe4RCv(n!%Hb68)LCtox69@&eJx@P(125ZQkP)B>Orn~v`ZVhk6 z<#Bs9AMotq&-q_`Rm;bbzpDG#-|zn3Ir^R*JEgfP(py5?s?2lnvs@>$cOP^hdw(Zg zpA?SHV^JUf+@n0v|2^-fEQ#Ln+fjTDH)nk*Zs1=(tyhP&cPu}X-}MvSd%3vWmBGz( z6&c@J?z)cX{olf#cf;H@;62X0>t|i`O~l8R$X|5dG3>?N@`qio+*+ROn%|lKjP5TE z2g^78!=i=m?YWAMYhM;i`(JUba(Q<)`}eLdWM3~1#p!l>^zD{cT9+{M#<2V~ZJ!@` z=QqUFt+DV2;YVk8=kU{Ur!^^;zWc1Jx$T-Cv{ypdUaLIgzTS5)n&BIH&)vs7^jaK+YNwh6aJ1z;-SbpFIP6E-(X%3Phc&2qx)Tr&scZC+!oor5DUqZ z90174K>nn(Ik7EJ~cM%n(e*|L->|*uUDKuO1@to zydquQJ$5iXbasvRJmbT6{=z-VLs43Jl3f(@^_emfR^uMFKK{-bXZ z#ee@8buM~q(!&*3se9>Hk>l|u{aW&R+=Tw-Sh$6}Ez%R7U);UJPu*$HPj3)s^+%0s zUN8ngwk$9nI>FpN{Rp>qe%ko$6ek_@_?5;4g!M6U2rkA5JcRyBpNQ`zd!C;iKS1_A zyfj`xR-+D0c>n*Nr`7++k(P4JQLju&4y)2r!s>~}Kc^c1HxZYahZ21dQ!aDkT zX>CLK|EB(*8`u7gYyZZze|?+Gxc09!u*6*W9A{Qz7k1B%hdm9}7PLEWlH6-O+rFdD zuE&1*fdT7N$?~pu&#^y%* zC?2>Yyh?xNd+Iy#E^OWQi*RqG`!@GR-v^(-5eH;^lKeDwJ<%OL8`<;s5cvgslKyFY z7GJ=Z@MU}zU&pubD87Ryu?#CvaGf(d$Te7xFA3|**hG)Yq^;x*ypA{V4qh?2)${zn z#^U_-KKReEA>_pvfy=PhSkmW>6&xTtQGp$|OFLx$z87LU$ll9W#dvP~slkkH3Nft)4Q zWMr9t%e*M8T6|GhJ=Y#L?UQG!^k+(pKOi6LpomO-*mL|7xzWEpH~G)!X7zWWN#QL6 z_Ft3Rn2Syg3EL-MZX9+<*vVY1PTrNTuAg*yD4J-D0z0rB#n_1}yu&WC;OIqR!?}yX zrc)P%jotpqlc9}|t=zUCPkUxH*5DcBV=dNW9kLM(P_H1DBNsVXg_US2l}FKvBY1CJ zdMCXDwWvWI4kP+EMm?I)h;nVjGG$N&c~HBtYOygA=IV4~Bq*WpQLgRf{;YY9(gN*F zb_dA)MM>eO;_g52??@JxHRKz(hCU3t=!)q?BYuRpu@X0MJ4jwit|dQ6YI-Ctk+xqk z9`>`T`fx+Ey;IDy!?aN1HvBhy5)WcNR;8T`e@|}24(vnk1K*4vAbURb&G9) zoM|l??Hx_mgKI?h^T|ez&`Fs7H${Xfun75L9*bmS$p4*BYBSFWuhCzR%qO0S_g}p#o@f{yvX758 z|95d%G5KQi9`--L=RrY>v$8FX>WZP%Ws$zW|3D& zOLqzL0nEkU;W1=m6H3v5ALHlvEndb24EvBe28;0+uEEntjMN6ka;!kk6ytwbiB(uV z()b^q!5XYxV*C&5upS#m8vnya?WIkljQ^$iSBElti!ysFxlI}V{`#lPi;VwS|1?#< zEknPJ+|y-`*(vLvl&yQoedn})upb9dc2XI4;m`ZH^p7g_iK=j@OZ(@Tva$W$Ge!?< z*FT>%WYSThVrg^#r7l*xPE)LJmmNt7X4*M=ho718-3PVGShH`U3nzOCUtCQ_5=Gk3$ zi@5C|Yfq<{qfWE_JuMt28@SiAZ(?pF@8Iv_n1>o1!Fue*Pm$f1rT|D&e>kHAxyZq) z9{Z0bkCwO6!Zu;gN-P`^Vp6Wl%f^?iWl$> zM(F2^!E9_mU!+UtU%;2}W&Dx9o7vxuk7EMnU;&=OH?bMt!w<0!C-EyB#s}r2p*Tao zmVAZ$6QfG&`^tp>P$uN27-zy| z7jeU9Aju_iumipP>AojM@p}XA!e3(n9>XeZM-avcjKgHyjlUnNeFn{v#5+TjS*gOi z^RjRUIhp;fxErn^&%$(7$)j8iI|4Kb|BY*wByDX%VU%tfm;Md-Q#1#Hb`zPKK zZW{AeOy*}ezRKUD_%*wo7{y;6IhTBg|8jEuL~U|@`q(t z;si?3k8$`J%JC-z*R2t5K7Ae;$9?SX$EPq2S7Dj3U+3pm`UrfT`4n>j9&ycW>^^{R zvHvBz&G<6=8uIVS-upga{O2-bF8C#erWpfaSBbBDKwm#iKNq*t9|r&ZKgW-7^=08@ zVfZ~yVx;gc$1mwW$941}-2Q>IG$OH-yq)|gs_2%nB-+sXFPHkqFUs)5$k6*K@>aZv zw=NC+IoO7qaW#@*@nZkC5&7NbnIr!Z+{`Hb-FKt|Y0?qe=xzn}X7 zvW)-osrHe(zJjb|u3~?Pdo@|J*#F%#{og&y|J})Y<_7kSBQru1nx|xh792q<+D7{C zJUY;cg#Wu|d-vtueMQ>6A!pRRAvgWru##NGyn5=rAur?J@C>b+qrwoSV?6k$7k2X=BR#xDA9?747nD8aMzz1YXC6#MB1 zP&Vz}P>zH23RH5d!XbJ!YL+_R0Eg*ysOQ#zMtT#Pr#T-1N9e6+PNYk^k)IBh6R;lUtd$Wvc(x zyW7bf%sbf^bKgboX5Q1F{%=?RlY6_=|EKJE>AomLd#DaDmz}rwWkCH;Rxnqxui}1) ztY)rhu)oRmgZs;8+ut+8@)?@XI=^+o#U=4B>Ob@wOiB(7>O%K_yUNfw~&iQwS z;y&r=tn@@~V=kI`r}p-p+5~r6k8)=yTzF^LviQ!hmE7Q(0&K)4d|w&#D_G8ycol}C z6BCt}Pmwuzh`xvXDZYmvDBGS_H$AB?`Y8-SCvL&V@nw7$1^9*f&9HhRmEUVH6%XJG z_$IQk8GnLhaEZ~l9(Q0S{u&SA5iCL>c132-c+Cj)CO)Y@c)0tEq3+xlL&Gq2$i>Ez z*nJlD)BYx0s_FC){i|fNF~k-u(+~SEb>RW>OXm2u-#RsXUq7%z-|XCng^k@hvS-QS z>n4Sr`Uu6?b>6sgS5kQXR_~@?_f>Q3yUnNn3L}Jdt^VPY>enH{tivZGKiobY+52q2 zcaM$kv7Vn_aNoi0C+v&V(aV_EFpn3`rJn8k!u~z;aQ7O<|9I}#3hx$SeOMphpYai4 zJ;CoP?$2?5o%<%&RN!aAd7a-X9rFtJ_#tl7k9`z33A=n`Quri)EBL7~eq5XE|D<7b zs2e^e)L(j4Xb8UxjidWQ)7amK=J0~~!=7;D>feUe@N#Io{cLEz{g1}S`$OkrM!DQ; zzqso3U2gEa`#z)n?i_&q51YRaLqpjk)>nqh!@;kO3KcJZFjRi#ics~~$Z)9U1ED&q zcRkZy&ospI-$maoynfI1tS~d9xbc3sdB3mXD*ir(>L?z(e>MGI#l=st+qJhaf7yFA zasQTZE*C#vB7Y|CX9+9e`9A7CP52wvf6O%reT-}@pL~n@Pygqv{*QfuE3rxgYPCf^ zd6Qz{nZ@=$*+;mRo^Q{@I{PEmlN++k^^qHyH<6pu%>j^Gn75MKTvJ4DXWl{XS}K!o8Jj6Mj3{VQ)utmU`i_OTuPs#TFElUJ^E76E>pws5Kjl^e4z&$F1e)v!(+( zupPO_?9In2tVH(6OJvro!-`S%_1mkT&u%T&V;%DPteeIfJcEXIYj)5iZ#AN}{pwI- z9ILL%njW+dShIspZXIZ$H>35eH9aV`7JJ_`X^q@}+<4c7OTr%Jz2vjjKUcA@ykI>qVaC%)x=n_T-*;qO2%zmIyBi-olcfAkDjx>wAzSM&cSx1qv# z0(T3iN%+5U{Y}ETjNgQPyV>V%2+J4h|LZH~khzQX|H)Ow`v0Z+|5f_`r*1U_jnkVK??HF#jR^XWetJd+zfLrR09*17z8J zee_b_YR2$TVchb6J|E@%^G$rq|K~8ZY`=@l++QbHvXl z#L1WOsCeCkuZz!@@FqU$`+P)vtrB++F+Yc$uIZO;{7?{EED(R z`TKSBUC9slE;ow*J8%U*pT=kTtHv$Dz05s7=02|ox0U~o3%5X+ALZ}&zVn0Z_u(Gb zf1mq*asRb2v-tU;dtAx>t9TQi7UuWyC1Ji3r4QHt#x;M(|7V5wTk>b*R-Lyk#?lMP z&Bnzy{{>0v`OM&4(v9L0sk&avaDe=Kq@;%^)Ot0r4pglLUXPQ!I!g^uL% zRBMcqG#1G9XN}b(A8WBD&DtbHYm`=_X<$-lEV6!~Ksi9xWy|+uEprXoS?pV(9nskZ z(H`p~XhAa$OdJySqg1DKANHmU3D2U0z6XQn9gneIe5Cc_WQFHCNS3!;7s|LlDvs8R zpC5{wI&uH2=sSzQTg1&z#NAJPmwt@r?*O~6i^q$_`)#iI6Zr?cA^t~;|2O%YM!%T9 zKQQ<6TPNJheY3ykZ@RR$4*$;W-%NX}VFluz;Z<_0dwf9{BZP4m9%BCm zdEILi0#@KT4DoHgg)4lYTkvIk6&0w)5Z|R2*ZWTQVzv0X{bk?VwN1*5fAX!CN8bzo zfGzltZ}#8tLEq^^_;1*SGdK_1xDyj_sjx=l0elVHP=h_F$7}eMZ}Bi5!D?LMyIg}W zi_u-WPB(q z@^1q2V7apc$jX-S;rG$|NjoK_&W%|Y$zQU3Z{$=8_$@3q;D2ZVeUu5mdw%S2(MH| zI_nzS`^0hKxUjQF9OsQw|LIR)&%oGFlB@p9SO1awiq(JQ{xM_2fjsqJi~0`-=@qCf zlKzErs8ITMP0iBLp;r6#u;;7Oe)U_}(7>%xI89`;XKE>z{%1Kuqh0!MarVHdEA&^T z|JrfktKw({4&W#F8T#=KE|We+MCl)`cnQ}^zn{fNrTs7AIb0>ZUyo1VlXwV^;w4-q z4;<-!JA9UW6rH#tD!Z@=32QmBPo{_Er<{*))<3kY`^d%0GuE3NO*j4`jhzhPnd9kU z4L@s7rH6d$K-SIFU!H4!7a?#tdE)sw|Nq!go^P?|%k+HKo~&FF!o%EWV-7yApZxm~ z#{W?4S!^*&?9L0}W%9R}ajEy!|J;m1&$wkw2wPE>uP&NAGaM{ZH=$C!T7^R`>MrbY zuaY@4!?TO-4}0eXXO|~CTS%Q}o!J4hIoH}TVH_b_i|wx>zvo^Z>gex~HOHNQi^Hew z&q6)D0gb|*$bCK*ViTHi2zA(l19;7|U%=bAn0Yh|GbV1pNAOYHjgRA#_$(g5*RUA> zi0@(<@~{qDumi1_j7#w_40R^rc+1Q`R z*D-+C5a&;~HZcODa1};l946pKB<5THyD%wSM_;kX`k$pq;s2rS-NU20>iz#WTTm%r z5BA6T9#pJ+DJZmQNl3UzvC^Pd zG)a@t5=bef0Z~$liWY5Ej&hFB?`ut}&vTyV{QG-mUhA{=+H0@9K5OmEy6=x;cOGE; z?>@%=-cKJteln3+UdI3SGZr+$_+Mlm+xaILBfxc52u0Y7Cm0KQgzImV>+cZP-xaRE z%Zx3+y$bhg{MFzuHcoAt`=2ocuI2R2gIQ~&X%lstE!1U_DBHwSS6M`zWj^KKrPNuL zQU0AnAOAe+GPs-X1I4~Cyq_`?^d6uLwU@HgKI&3?s8j8xjPx*NrCpSnR#SG0qYOou z|6KoxbFD|$bd{bu zy_EWF*+J^j`>C5U4y$q}_2&DjM{hIZ?Wi{+>yXvksYmZ%yxJm5Df1bCfV3l>)NRwK z*QR23;Fh_CF}u`()8m=PltjIK3HA2H)Z>>@kEbr&NSZc~wywqG#gf@HH)-EY+B9t; z%}v^yJb*Urout3{Hj5X(9URkpkhFh+@+x#;ALLnO)>b!bEcTK9q4TqA(*!Oujt$H= znW9%1$MzoM*e)>-oHU#dcahG&AdP3>5=?*Y9_wSce+}-1@4ycH+{3;D*i#_`icXML zNFN*m^UNzfMA{v}eS)-mk85v+Yw#l1BGjGd+GD#W{u@>`+qN9O?N@TGl54J2O2)NH zW!uqzt(=i-m4-}bUk3X!ky#h6RW=WU9LObo@*saZ*FO~Q;`)bT>?Kfokn11Hu~$GP z`>LQCdkvT|)OAolLE1yZG}k{gVQ+?(%j5;LVRwUveeKYJ-3z`=Cf$hL&Gi=|4@4=Ih4mnT) zl~4yw;07NYf|ubf_&faTzf#77&%M$pt6-a;{13^~l>Z?Wy91n; zDgQ${_6*2mUlwFz&w>Bn^Pk0XTCb4*D_KWz6KfD2A^%6o|9PxEI8FY~S;W0a-+DLs z4?V;wHvj7c<)1TVY!vzbF!@i}D8PLfyh8c^66K#47-O}E@;?kc{2?vk*|r95femmg zd>!tCpMVnz!39wmhPTQ8OYjVKJ97=Ppakl`4ZZMt?$dYSzu7_r4=G%0=`aSb!k^)9@E!amGr%at3^HPXl^y9EooAH2i8{TTwt_RX8SG;o-~rkUPSBRHn{jv33siiWHii9+ySv2v=OeTgFlM5H zaz`!u>JBYXHL?a8wllT@>Q_={N4D6RCzZiG_=^$>(j73fPE19pdW(x3(`x0LKb90PL3JdH<$ANT+07*Rp6b={GYih>YvN}pSiQ+Ul`k0wv+y^ z{q%n||1bSt^jXzixJLDiX>rA~{=rK6zlcW@WB8kiQwztoPBWiuAN^nSU3uukYG2BH zGdfhfdzfz)p#N(Q{a+W%H*)CzTFhJl=!>WS3kF~i{EO-Tf*^z-%>597DEGn;b%jT{ z2E0)4(Z8t&>E}9%KsFQrUEQ{6_!QU6a(EEF4Nu_r$B+g2uo+&19qil6by@+v5QLZE zckmwk4W1{wZt%czj`ywm*sdEZBID+m|sf0+|v=`Hy$MV+-@YkZEkEkjbd@ceV|{=35aZ}D7pb4~Otn=7-%W*^ss z`R-Kg`_kum{t-riu!3B(AqV-7J>r_9XVCp1WWY}N4wOJOJOo>LrhFMLV1E~m!W*y@ zw~s>$JW5`ctiDQ}=BmmzT~#TvV9{0OLlN7BbFR`id6hPwtCTyg(vEml74xpD9QP_@ zB{VO+N}tD7=DJ^1%i^nYv9BJtM#5;Af0eeLt6cL}8P9r^KL4wX*Sbo(-Bq>Y*MsaQ zj6N9TxB+DM-m82Q=&E}6TvZQ5wp>-1xC{|a6avey(kF6NA!Lwq+Ah2=J0xFzUn!7! z{(U(vzAxv@`}F_3Pyf&Rv~gUe{rjr2;>drli<~XwKQfQ)R??sz3LmEYb3f&u9h83# zQ2u#=@(=QVE&t%Bb{qLmJ+YoS4F7BW-%k5Cw1FEu&<-8o1s`-m7j#1p^g>_!`|98F zKF>kw|KJbs{)2yhKf8eYpSph(IRvj0_apFO?#1_!4rI{%G2Ms!8N_kVy$q@FdG4u8 z+&edO9b}RJKL;;#vdtRmw!b1@#7_|WIQB;P1NL6{7Gc%G_i_6g+YfQBM8Z77_I!?c z47ZKA-2yh;PO|SE{5P?0H|IFZ_HW>OxcwKj(6_wc2JUI7r@wiOcWyt@&9)bM|1a@^ zU(*czv-!N|;5OWwVI6FMPlAnm*bcGv-}Z9FJT5QKo{3hH`mku`h8cfK|lKg?2mnij%(C>_vJ&R z532wQc{UY6vAfVJA#qB{tTHca{Nv{+dQU?yY4|p2Xd|yKt)SfjeokA> zx|O_dAPZ{XB{;O2YX|(RdA~uBfp#GXV~;?T^2g9YKkI*Xt9uLY$0Xj5i+Dcr4(#97 z!}!o14K79TKtOihxyg_Msqja_b$*sSBb;>nWL)l2=A|xWz1Kw_Mi=vc>C>2|97NeC zpYs$zA^TH^YjH-ARYJc=X$j>ZC=ZzV2feC1)2pfs`cNw9OMzPKbz2$#a)|doG%W8` z<0#`_3V8lQ%hnz}MtE%(dH-MH{eOk`Kk~Eaej}ukMrqJh!~37|MGtA2&vsvcIS9xB z($}o9>@OjG@DoIa_K`jZNS_x-ALJ0*$pwv83fO8W|J5{F&OoD;meXjZf}_9D$}DNL zve=*A-DqW0G+MdTr}MnbgJORUb?QROiA9h!b5j!occEq*>pu zd}1EuKjyjPx9$w{-odQz*AO>PjjQLWemV2vp#@r3QvORaflD zi1Oc7)`44(FvF_0p%74(!F#{YkNO*mO(+|P-==Z)vKRDq8uF&s&o__Bc`oS;I z4-SRPPN-<*31yOIzvg*Sy8ML7pm@s(l_Z^D-If!og6X&GtqWX_ufFJD?G1-jy&tz< z)LY3+GDz{B)E_wZKf&ofNxAW)Qo#Y4yH6+!rc>*!iyWVeJrA-uJ_p{zZO-qwuj=2> zhmiw|xFq4{Fu9QH3HdGBBP;vfUnA*H=gzM zwk%XOq-vH8!7-^&#oBw72NtU0;6lne3suKDe6`4$eG3`uy^wDr zE@W=NLKV<|n~%Nh;e{&QW!8vTsA6cJSV-ND?>HV>sJ8Bf>g?w|Kh2u?Cl;y$8Y>p6 zp=Kd#CM;Av_SPK6mSilXjJ!}y5F*?l?tv1%#ef`|PPZ-))fUnz%(;343t1atA$7!s z>YAXO(7cek;X?Jh7c!3qy(Sjuzww`v#2RRfhjwhcMyZfi!1!v`8_Oseq77t7S@t1i zvQ|va!E3no7bvfPk#g-}74Wg_{O&N%wi>IjpvEfkhN%ODRd!-XrBE?Cq;jb0uCprL z)Op-dRUcqH1@jwf4>G=d&XDR^gRHTnTMdqgn(YzlJ`vvi5w*Cxsq;kChI>119_-%t zBI?MAsPjTZzDp5xFCS7@O+>xhCwK-#)bENgo+6?_p`{BH-A5u_iOPPW)=J(JRmx~osRdCvYNB#BN0pWnRl2vp$~YfY=ITN#t3Rsj z#f4T*NmRLUJ&ft7wel}SS^uM7g=RLx1O|1s|k#@Ms|$9&fR01snH z+Mxrx7kpmE=r=R}gZ8TKlH=;BIj&yE>(u*>Q|~*jf$29i2>uIiC;-9BZzu#|>=B3t z-p~-(wx5(8l6P?*Ln?O1f!E}G;Wecp(|5e4jQd|x=7E#6|DL4%_ayDVCu#pZ$@s_D z82|VhePpkxaN;!;LGko!DuGh$Wl(Y01%S zpH>4jo;j_i(bH-^a#}6O)+?uJ|2WNf#M9g(r)mE^aPbo+ErbI*R>|AJfD`b?@sJLL~chW z5=K1sMA!mf!|e{_mtZ?ILJfR}WBJ6Y?IHLOWh-+@<^s2$jnAyux)<oVzj_H`!S zptO+>>D?yUz6!2U^T-vYETarq!J2P5-2afqJyLK3bwBLscGi2YVJ$ewIdAU&Ym|49 zwh->6f^D<`)bI}Mr;L+Cn@2`S#XBkcQ_d=d3MhvvsDv8o4b@Nwwct82hqZC3`#}RV z@99y~CF&LL&9;HMr)Jwg+mWUX)EhtB2HJ@+qcS@>Iu02ay@w^x+|M;9b@s`5Dr`2mSX^Cw5Se z|1EW;CdSGYU}th`={tOnu)fxMwXNLBAn!UOtPlDNo_E#64;{L;lD;=kw&48z$jD|b@{nY#DAL+yWPOj5C zKc)VB`c#OQStH9uJdN!C3wSSN(i^K`pOD2~7 z&C)3L*Wh>D>%T(21DD}b+)H)z2S11X-FKF$+Q_zLYNcPi*2@?G)(CKXk@C}->y_3- z-wN@KQf@Hcl?{*5SC6du;(AppQFX_dZ*=ztRWSCm{J#0BdWboH-{$)`i~%ivlrrHD zSR3&P?iX>t@a>u+)|o8in}il|SP9pHjr%hJ`NlPr@p(4A#r=OC-c4Vo-y{D3@4;W- zzu*e!V>~zD!@z67wh;aeJ`SITdd3U5=Fs;Kjf_`lns>bz8fd%0>|6F}mD*cXwy0U3 zpd9u!+*lUMmI6*Ff!g`as+-fS>iA~W;IDFdv#RE}Rld@#3f#)^(*z&q_y95tr+{gb zwsY_|_-~j?|LI5I8dwC^!N0=5{8kM@-~J}`L(dL})qB8Ub?s}SEyH0g#n1I{Gu*+m z@($#`gMVqO0?@L&mHcX^%+Rdnl4dm&G^??uS@jp1yLX$Lc7$fN z?{=&0;FR3>x7nLyU(`bTmq#gcT9gWo5C~HXz<<=JEywIZD zy)DYy)1rJ=lL}V0%)bAS#mie%a>zqlPz&_{kIFCb{@dnJWx#1w5l(fWMKzaNRQp~F z?Z7RnU)&z z@Ygx7g|Q)-R`;Pys|VS;tVMl0NyjrS8W{D^7uQ1je~SWB9_j%t3LR-tc)CRq+@ljM z8X9fU4@pZdRomy`>+mhOjdbR-(l$Q-ZTm2NZpVd=g@!3eRV66rG#vE0?Q5A?uL8d$i{T*)h6&~u|9Z~bsugI z8TWMDs?#lA$F}d9?YSA&t0NiKqu74{EJtK}8lHo9&oT~$_Um`Q_-Xmqu2bL_>lNf% z!aru;t3M#zJF^JiY5fP=EdOFlg>=X^+ZnIRzOsYS_PVwzA#z%2|?aQA3vE`x#d_Mm+nN8HZ53a%TP0KVD#d@< z`Fh&Fv#pBV*;Xa8DpJpQ`UdgJvDNPBP~HA|){CPJ0U9`8HS&%dXcys}Eu3eH^m~Q& z05{=#F4t54tyjm+23?DrZ`14QJW)^oPrbT#cBlv0OPGDTUgz7J9U7Q#vj%bVudY{M z4{_m`(9U`>M8Xz1m}f;d)obWVwq>hup?{8*>~<+-sh69SvQNY%=S7dwmgHFJH8v~5 z-CmU<%U*EN{^wE!_R3u@ z`mtR+M_j7e;!^Eu7h|kks^8tned?mE^Mo3aO*1Yv&+)1S*_z~4+v*c?JG}B7v|H_m zTUBR-dttr~BGYOlCeH!s&J&uC!2RxaiB2IedIRNUR5anhvpLW9cU za;^N!4b*uWRET?#y@CF~H&_qFr`(EUD{py&(jnuJPnpQ9OFpIAxt``V$Vr(y4QY#S zkbO&ol2&U*D2EH@UpaIT1$aYsngSrbE z)I&Ub9SwZ1sDV0>kNMyY>cn3gdF(#ZAkU~z?YK3)muof8sMeZ1t8G&w`JZQbIA439Qytq=EHCl%?P#N}oM&~tkZg5t zZB);}M)e+OWbK?htA8rb8bA&bFaK_r0v8)~b`{tEOrydOIn=2rGVT`c|NR}5f$M4i z>`+QG_kYEkawKv8Ph?wZxTRm{P)2iyGF=_YD(GPRcZYHUDOT>~4&`mJS^3i)D%fMQ z3h`I8i~HG~ZIxWissW{7jBjG1H++{8gRkQ1xz`RkPn_)gtS>9jt-ulPjk~ z4JGwzw0EcpntATDAX_6H^!s(t{@ua&s1CK4bg1LK4tWXR$N4(XchJ7)qy3BffBT#2 zMfSNm)W5Y}1IWPx-2Z#uRA988_Q7l`L^$D19f}<2P!!#Uwsuh1^va%*Y$bEAq%5*q zsTp4Mx6=;hRT}qH`gD$!k>gb+@06^Sb}Kt4+sf(oDi?ow+?V+~?N-57yH$uRs`09L zs+0L|-2dELWjnn*|GcW;9<4-H?eMC4mzVW_ysUrVRoyl(?SEdmj(8dW;H3`WRnz`X zHDBWXUplQ;WZRTi?s%6x$o6^M|C^@eQ(Q4ulkVvyOON| zo@Ik)?3Vwa7ya#4kn@EuyA+8Dcc%&ucB<$=Cu4EQf80ywcQXH{Q{{H@ zA6a?fG<{y&|8qK3!@XCFtUJN|Pkr6h%>BPB&uX0MRMRfewtH>4Z|MQKkf6+-tf`{>MZM1#;LMh19-EDH%o8&~M^*1Tq{R?Hx zYhwIsx|OxHjqyKi%IR-o{pU92+1qIMZBxNSnpK$5rlK9GR`Em|{a>k8DgMf~H?jV8 zx>a$cO_en~|MsL>)kiX{n)lk6OVp;i(KgmYO|@M1RI35m_;4HLbK3S6E!1t4nFrU= z{=J2^LEJpM+qwQ}6I@2yAkv4Q&gQ7qb&$3_=$TJjAM`DyZ4U;p4=!$(e-mwkSE5$% zQq&4z5Ay-r$m)e$|Fjhnj%{@_}qA+FR-J~_a-IAEnMrcHFtt;#qNwla^n zDgO*v*~@7gozAdw<7g{gNt@b^C0D=10Fh#xt?}YpFpO(;88Z$NX|SD>!tsa_oavTsfYR0 z9@-Ns8K>n#c5>df9a)xpzegVYwQu7% z!fl176IoXC#Vo7gJjcysS@i+8T;RKsWp$2bSsiCQ@=j-2o&vYpp{?63Hw=&#gWXwH z-vzh&p=a97I9s>6Tpo2p^pIOa5T0=>0)#BkwZ%G;`+e_b`^8^84rP){$(Rm3d`$?p%5xWMyoPTBW<)Dud#!Zk0gcYVr;8 z=Ve(1Te7U$y_D;qdNKJ3m2p{C)sif$d=dFLKg-JCR3GN~UWI&UE$wE=+MDQOK+;TU zI}1%br+3`bAdbW$>x&GqSOB+PrqImVcz!=DFtl`JDpZ$UJ zjDegVuMmWBk1UH1{ zZWCkgkb>uvXzPGC@bgvt)Fo2S#=f58_^hbDir+Nq6<>w>VJ{TGv(UnQ-#RCOb*I;= zIWVe*?zQwe&?dKgtz5t-b8S`l>)4V&xjR7~Wc$hlwKX#?k?rrW?*;faY4-(*q=6gbI zJ5u!-+8)_gxu3SklC|{5(KgBc)5I~bc$FUhJ>|pItMsZrVjcPRkhO}u{ycQeUq!## zD(2U$((P>b9a+x&g5}IFSgygG<@yS4?q#c}pR7_lbnIOvFZf)`nV+*tUt_=P%5vJb zm+KqYn>MXd)*8n2aK6^5jIgq=2x$>@1=)Jf~_7|4(JX%is z{&GD-92|jFa&m0i&gFcIi2l5Kc-mnd`699{**emZ%r~}>|HAf1ZPt;$q*$;11-Bd6 z{x|N!=j_y{k*`u-{Nj8$?pz?}iwXJ>VHd*p&@B%ge}?@D>^|fGTnp(>)2Gx&I~w-8 zF4FGK7~KN+HQQwz^Xf>dbrt)6a@|^7Pp>|~Ic=$Ye+-N~OFZAk&BT@X*jCeq4au7r zzX_?>9pI#|Gp&Yk&&`%TMmSZ3-T7_lOW3~x&%n>Xwm3=l3td+7vLxp0CMk9CTsbZ{ zEay@i-+yvi>06SNu_uZ89OK`n9ai?DGWso(l)IYoZwHd-)1&_1@8JFCw2A@_t9Yl) zD&c)mI-01mD~T$17pdY%BK^RLsydUXYGlnmr&W81Hbbc2nkW}EKqE9k^R6PbAX^6+WND`y`3?$z0}tk$!XReE%xVN?wwv6v9oVkJ&Mk%6(a?v|Wk3 zujVOZUn2V3t*kStR(5=vm2)W7%H3qQ@{svwl2kyLh4w-fA&ajh(f^gCQqEDfEzK$? zE)`JePU8AbqWvpLHBgJauDek6yOQLpp&z#-&1%HnbjD^iZ%aC<7g`=puN|X@OpJkq*%TC*HedG&$Ds^{g<`W-O{Xqd7o!) zONsmtYTlq=bFvlLiVlpc7z(6X(Ore~d9PRf-t}snq72FL&HL7?30jw}R|~k~)~gNL zR~GQ>UN7$j(va;=@R85S3>eDgJ}8Asb3bsurUhk9oRvf|(hT}9UIOw@;Y zH`UNj*|7QsU4#AO;97EnK7nlAbc33vR_N2%Jq0UtGqSCF1>-g1F5pT)CM! zf}2&0EMdD8S$1Tz$}er^J+oPr*sIQL=KkNT8tk>my2xhLKfGD4y_?l=aI+er341fN zuntk{vRmo@yp{UTt@3QTRqfsRwD(lY%eHUnt@;_~@7h|c?jzOeIgzj4L)El@RI7g` zpZ?Ek<`!4WKT)j!_TY3i^~-97yQ}H9s8-Znt)b>>#saL-VE-ERU0S1l?;7e+aq4xi zQCEB%^Gf6N4Cin{GE_n^E>5Avaq`>OFzzUhbqd$0^}-rua(y(9&S;)Hup-Yn0y|r-Ga~ zy@dZa;4;2Ndu=3###PWEkUl)H2d z{q%9lo{Ccr6ca|)p*W?_#L=$9#W+(<{jZk#-|wjZ)l&bfC7ePzk!k&f)c<}*{jZR| z&o40k=`YHj{(^EwzrgdK^^eGddd-#VPR8+kneod^wraZP6KZBnk(P%(t=2Z?W`1|E z=G{Wy3@m~X~RlT_VoqghbbK&4c?i2j{ zigIou+k@=?72DnK(2mdc-H^ZOx77b`$|Bh=_a6jvJKFs=^$YQoj=1_k<$oid7zMb_up>ikdcium& zYWCM0dPlXaolpm+_+lfE)=MaCk2U!Qy&%Qtn@xMsi&-4AhnFWmBVC>2?-|ss? z`VnW>F4hmFuHLws^+PxD{m5OzteHj4^aGH^ReQi1yXY58cH4Q1D^zy#Ez--Ov0^ z8l`)Xy}W<>4luuS`VShwKDdYZp_l%^{6FT0;y(moj*UPR`w-Zs=Tgs{OTA;RQXrLb zkprCA(;%IF8IZ}Cye!7pWg~Oq82ixtit?7D|Dubu|Dyk1^oQbu=>G!a01wdTyN_{y z_s>(s!;Aw&R@E@>&%wCA9L516>kd-S%AkK68mMzM9vo&&7h@oD82f;1rN6yx?;*Ld zdvI&tIjj!+dBMjqozQjpMRi|#MLq8^1_Ju1&zZF(24E2U-Sp8z5JC_xp#1})Fa(Dv zXa5KHK{gC?Zw$d9?(t%_+kmbKfjN64z!=E|+G=4vi{bqO&uHg0v@^*L}o`PqgsGoWLP(Yu5K9rqcTmh6|FNUg% ztP21Yhkm8;G(H{hxJ`6WMi$`+pPjA11i} zp>LXN2L`YYf`7^DwEuDqk=LOkTr(%Qc4!NUBF&mo&yxoKO`7B(caUD~$ZsGAN#`-_ z4`Tl|?1uO8_n{j}H|*bsr{Ms6B#AL;@PFW1_Fo4-$Nfg*hCLN~2IN8&ZlA=h71<6& zxRt{&_Um!4!G03^X;{H_1KWSa{#ooF;a-0q`?bgp+&_!F5%+cG8312}Ti708UkKJ= ze-Zfz?xV=3kUxdP*pEUP_7dI`#ZY#TG3zzcyvrEV9{)0ZnWyQ~V61vGW7eUL`G2); z#DMCW078i*h6p| z_95g_Q(nZ+FW~}y{s8B2|8LqX@4(NsRDT!1b@1sp@*DX{xE?+W=Q!?0W^5ehMk@;fp*Ikbh<0SK0S<_&fGFxZjJtm3^Ji3qufs zF&Ktd;3%Ag2l4wKyz_pA{T$flbN@r~QreFo6}tnR+*4_gjy(f1*_Snc8Ra+b|3%#Y z$h;T0|B(f37b1(6asMMr*e*quZQ=gk^($4(;r@pzsD>J-g}SZNs=vbhkKYE4X+$;= zMl-U7?N($PVYrbVw%d^%?DrylYI`VQy>AGo0n z+Q9?OODK~Y))Dpl147%7m1qPmHP-S;2NWvTA}dH?pLHGI;@I@e8b}k1V(_ zs{ENz6=hIn?|)x8=b4iad5{Ykqs-ZdEXV}MdFJ%b|C7>?POvjCz^s*!a+x{*l=J`0 zHL!qd=tEr3AA#?1eO!ku;#v1ExG#Z6aC;1H!~GL%mm#a57lz;l+^MCxQY7weas*&+Hr8EeYe$2?^I5!RnM!8q76jA=v`vt3g2o=PvXzKi2M z-hYg@#jO%qg`aBn)llf{Bn}}+BOX~WgnbY3$;19b1OcpJ_G z-xSwx;J0uAeg}M8T<^d|_yhbA-i7zzPw;2>3tR%ze*RZ5V}HpC@eOkQH~by0z*Tr3 zY+#3ENP$#vfD_Un9Wo#jnCGu-$bnqQgM27}LMVb_D1lNagL0^VN~nTrsDWCjgL-g5 z12jStG(!utLL0cj1MScOUhqLDbU`=tKri$`KMcSi_#ps62tgPk5QT4XZRs=m%&T@t zfmCpS6Vf2v@G5>Vji2-*$(Wgt1x1F~-lEvvqS)SIT#I9_#WC07m}@JpZP2UFnC&B+ zWtuRivklV)m^p@+Ef-sE%$A2OFJ{ZfmLIbjzXj7>n1%Yxx#S9CRBE}F^S@31S1WXb zR%(?#tIz3ktn&Z4__fA}*%FLxoiP%Pkz|bZ#@JwtjmFr7VZ0@nql{PM(|9!gj5p)U zc-jiL!yRxZY=bYsUBE{LKDQmd0$+u@VF!E-z7F@mPT<1>pSu_CgZtqDcn}_fUGPoV z4c~)D;8EBE--pNG2e21@2#>=Pun(Sur{HPW56{4}@Eja~AHnnRV>k#uftgfW%uJ4% zDKXO#Go3LrEoNrK%&eH19W!%dW?sz9kC_EAvk>!bzX|hgzX|Pazd82pK#ehj=GeD` z#l{RZ8Z&H}n`^Et-<`TQJX7Vz!U&z-NTGG=w1F}*FAzb+(`@4Qn`U#~-=-P=zfJ2k zW{=C5y(R@Nq?s80uFL54`{Z(CW;GeJ#>D6Mt>(PH?=taxC%MCzC8@?NZ+fTx1Ixv# zn5EXF)jN&msCT+d4BrWuxLr&!Az#ck>2tA!@KGR{UE2MvxN5tWL*_dBBI+A0OKhAKMWh+mWDndJy6hVmlII zI}&0$)+v4+JJ!W^tcy7%Dn1dX#F$fJ%qdCnNjN3NoRVTr>lIJmd;I#C)B2dx2E}i{ zX+z9uL(FNT;y2>7G3K-}=Cn!in{e6`bJ`ShLPf=67{w4oHm0;Oi3}#^$K;floCTAB zF$pUcOiUpYRDu$2GJPcpYxx^tb|=IVCm~)5G2aQXBuT()EIATl36Zc)39&>-h$TWo zq7q^WkPu6Ngd`=zluuZ%gqX?+8V(KMqQbJ6r1R_Rgm>d;5zlmP#EbFxH zCXwFj*77&I*2VLWSrW94y@^_vBX_Q=jdLuwH)z96BD*)lA{9%D4Q6L7-#2RGO(NrC3A52S$MSxYHr*sL zKbB0JjB_miKd;Y!ehVd+TPd|{G5;+TU2fBD%XQmr`l7aK>;2mLR(i=k-LBjB>ki#< zr|x*_X5E3?%#~bBFHM-Cxn&_uZ+xf2g}(8q^N$ zxLrH$(2hH`g8+6A>W*({#{=5&pmt2U^)-F%EBe|G^|hz;b={+TzN~xh*#Qseo^R-$ z`?iS#cIq4Y#{cj#U$Khk}=Z`<9vk1+4sq5JiK9@x%521ZQx2le0`dhnZi z@Oyel4{hhaUAy#6eM{fg!}@o9N8i6&Dwp-Bk-(tACBsK`rhtm^oSn0KzZag6 z5t#e*m>wg@ALs|@_JiHptG(tAKh)kQwf9Hb`(yo3kL&TRdK~>8$J^r%=5~N#pbixN6LL*Cg@2$xm8boNl!kYCm+<4Kh%@^^yIU8N>4qYr}eZ+ zxTm-3=`ZPNwjb2fkLc-VwO`NZnca^XVqtwC6ZEW}eMZkdr{{D)Khlr(@{h4s&+Etf z@m~IWbx=RiPxT*qK`%AvrAEEfte0B!QiopZx&yu>bQ-p4*skGZ4X0>0Rl^PqJ2jl9 z;dBjWXgE{DSsKpPaE^v^HJqp60u2{xxJbjr8ZOarsfH^wT&dwI4OeTpM#J?Qc4@dl z!;Knl(r~keTQuCN;WiDsHSE!FyM{Y7?A5SO!<`!L(r~widonh-Bv&JO8Y$FB zkw%I&QlgPkjg)DmTq6}4snkf7MyfSZqmf#T)M=z%BQA|JXrxgiO&V#|NQ*{VHPWUL zw?;e~Y1c@HM!XvFX{1vlT^i}uNS{XfH8P-)L5&1764XdYBVmn1G!oTlibhj4>d>fD zqiGsV*JzeTvo)Hd(Oiw@X*6G>1sW~VXt731G+L_BGL4pNv_hkm8m-c3wMJ_+TC34I zjn-?_B}N>NHfpp!@3!9*wqZv_qp_jrug&snITtc5AdpqrDpK(`dg&2Q)gU zQNKn58VzbRq|vZ=RB>~?oU51f^>V#lcIoAIz1*pnyEJCgm|bJZ8q3jGuEz2-mank_ zjTLIFNMpqsE74e~#>zBSuCWS@RcfqCW7Qg~)mWXz>NV!lScAqIHP)iBR*ki3%&jqx z#@aR3p)s$ZT280YHKSii>n8VhJFsIid7!WxTcEUK{~joUPC*LbqVQ#78cafim8 z8c)-By2dj!o~iLHjc02-N8`B~&(nCm#tSrFsPQ6=7i+vkPoI@hXki zYP?S4%sFY%c&oHIc800!aIy}k=y0kI zr|WQr4rl6cmJVm@aE=b=>TsS8=j(8R4j1Zhi4K?RaD@(6>2S3U*XVGa4!d-?L5CZ4 zxJiebb+|=`TXncihuu2t(cyL-4(M?BNxhPC$AQCYvR3p}!a5ewv8av>Y09Q4yQY#gm7=LsO*u5>)Kr?L(lwQ-sVq%pYbr-m zxthw?RDq_7G*zOhQcaav(~V7wUMCj+g2VA%C+`#NWtb{zjI(5ika}Qe%`GqvDM|V_>T^Miu{< zG~rb9HybrNS!-myFHfetX^+`YCL>~YWApGg`+Lyk zO_$zm(3_2Vvq^6@>&+JaZ?+nv!x&y;_>9qQj2?_>hcTSSNHa#dF*1yiiE-NOIBj;E zHakw|8>2)f$kyqUH``1&t<$Oe@Y6~|21cLWY~ydv*k;a%VZv%NC&rkyb6lG_f15df zn>l}*38KxMy3KfT>rD@T6AFg$WdwVTV86!%jWO%NJL`cncsN7Mvv1aO?0g;*PmejD zC$@9i5jzZF0zsGqrya2)5GHsNC69@cXVw+BcK+tLb`x5=39a3P)^5DE8?WugYr8od zUmYfF4C7{mah*>6j4@(KhX0sa7%`u-r)O){r#Z^l2qdN>Mr_lZ5p7MVMi?Knc5G{6J8O@{ zq}QAWW7a(uc&`bw*F+5?cJ!=LY|T2NY;VlJi7*#PpYend+caVK8Es~T@nKHUX9Deu zxz6Sg_Sr2$>ocMCnZx_}n-DNQ5HVL1Lca;2pY#rxS^r zoH0JIf3OwvVf+pno1eeg5#Vo5hY{PHjV#-BCsa8gY;7#|38C|j`uOyH@pYcM7V$ZVOQ%o#CaoTO*h}qq-6UOw$nDu9_eLNuE?3}fjux5`kceJt3dNH@#tkP&c zYbUNTn^V(H;}^r67GVxC0v|?jq;qyp?0nc~_r&ycnzK3I>^4RZM(kJuHphMtKF>Mq z=eIAxkRWtw0y{vphfCQjVF#vs7VSO_L`4kRTxGN9?8rbC*JYI;bgY&w;qQ>i-T z)TuO`O4q4uoyyUvT%9V=sdAli=~RVXWQ7>RyHfDXGQz0=$RG%nCNhw?Z9V8 zLri$nb=FH5Uc&Gd>uhHxu-(aaH~V^RkOs!izFzkABm0fS&tN4~Lk-mGY#`^Xzei_- z?Pr4(v%;mbp`5cp9K#$Fv7KeTl(UiKv(Z6B_N*wF6{WMHYF2cgjc`x|f%1qbQHkcq z4v!YbyhN*Kg=9|(euw^dS(yp{ij2(ahS2SXeOmvwDN-;k|70B!2wQ4gLKG% zOvr+4$bnqQgM27}LMVn3s00&s4b(zCFbzJn0UDtd+|Ujk-~}IaLJtgR#zDLsoZrFu z9A!`r6+l=H!g3IngRmS;&m!Ev}b3Q-$=;xe%^3l(E{p6#c zeDs@eh=ZSa`_bJ`Uivw&pY!@je?NNo$y{{Z?2&_96w0rU@`e*pah=pR7;0Qv{eKY;!L^beqa0R02# zA3*;A`UlWIfc^pW51@Yl{R8M9K>q;x2hcx&{sHt4pnm}U1Lz+>{{Z?2&_96w0rU@` ze*pah=pR7;0Qv{eKY;!L^beqa0R02#A3*;A`UlWIfc^pW51@Yl{aKoHCV>6{^beqa z0R34;Zzh2LLG%xze-QnH=pRJ?Ao>T{z3HTp*9ml&mejR(KCpiLG%owXAnJu=ov)MAbJMTGl-r+^bDeB z5IuwF8AQ(@dh$q|38H5ZJ%i{OM9(042GKK!onCoA42~S`iIazg#ID) z521eu{X^& z@-i)JF| z7eT)W`bE$$f_}V_XLudYM9?XMJ`r?@poe*%Mu~frxJQY5l(=}6y2ie7Dcxxx<%0~if&PKi=tZ;-J;}O6#b&)T@)Ro z=tv=DCW@X>@-2$4QFI+b$02kaLdPNW8$!P!^czCAA@mwTmmzc+LYE=pKScb8i2o4r zA0qxk#C3?c4iVQO;yFY-hluA8@f;$aL&S54cn%TAA>uPce1?e65b+rzK10N3h`0=e z^_DFkil7*J_F@}-{MvOmXlzdl~4sfy_J>&xsa!~ z(vu+-(jWt}f#cFSF1-m_pq>8*;(yp?H+*lU6KJ{-F;JWJWzSfEVzY z(FtA94Shi5GyD+HTbVZC_)Lz^bU-=~MrH?ad?v?da(otHX1RcHvl_tz#4!thS@Xh_u@e(J zv0$RwIXf}1v(H`*l~4_}P!A2z46WdXF6f3{=z{?mga8B~bk2^#_Ao?rE;;>NGHH;^ z85n3}#u=SUCgfz&Dw&Xz8_uN^oJ$FxOCiC`M*F#xk`MNtOX)qAQi<#A9{jXpYlJ4l zxs)v9BboS^4g8Y!$;33d#c(d!jUzVVo!n+PN17ye#!g1gu+3?cIUh0Q+CG=;F~ovI z6=UO!BtWt`YkTY*#s>!waLR0i+3=oAF<#lzVTd`H(_$k^MpdujTuSplJ+kqEo0sUv z!ZYVF&gW8GvGb3hSbdX z$LH?)fA{&nx77T%vwUO6ZazQPu*bh+x7KHiz5L&Qetnl@xxenO_kZ5%j<@cuhtKtS zr@uDf*FS!JSAN~~XP}>yF+V9|PrvrW|GssvR3uZ=c`F|K6JN)@SAC z&(c5E&yLp5^?$eW^V>c?W{q24t#8(ZHTk)I-Q(wG|F3Y|w|;#3*WUe181Mc&KI=a{ zt8dvuzy3Y@@BeoB_?-95{o0kEw>|&4eg*mdj{L)8_`e_E|F^mS`+@%abdRm^rO)-> zCog+*`CmWgeiA>o=ko^q`}6zt-&^+bxX0Db`ETUqly599Im&)XgQ`Rl7bZ++c||Jy$vhT|K*_TJ~d^nm<)klDTd^Z6t7 zxy|14?eT5-e9I4mZ_@bI=lbLo*E@cF zTR&SoB*u3?KlIt{$Me5`{JOQ>e?Ry7=N|W#AK~l&-m)ovWb}RB&)(zIQzFT272-m=4dt3KcTxj7gAZJ_^r z?jMh||8DtncYH?MGoQEUEf3dm56W>5#b5V*@4p^HA%EZVz^ z-oNO5OLo<=pQbIpHMV?w*8^qWtsf{p@b9BZOZoA(<#x_|-noq%*%OYl|Gn7g{CjC% zT=`~x_ak-VfpV|^(T|;Dlz$y}{OkGfzn)h-`+N8N+q(5f=lRP-=Xq?k^K7p^J3sO4 z$-wi}p!3|_?>zUvSofXhnT^i#+=}JD^S#dV!l?7SY_05A2bQXUEOYj+vhwGe0{{es+xf?D+WEF|3)(&a>mhXUB)njt@;= zTFaIu$8*n))t()HJv-KVcKr107{_{9^rl0SXNMKfo=!h|+WYJ&?z5+~&z?#?|MQpi zv-9F9!^>}@&Wk^wFaAuvcnrLFWWM-Od-*!Fe2&vQFW$0?i_r*MpY zT9?*S=Vij@Pq^I)pYQqF%fyPcYOPuBW5U-@_&UvZ?mI6%U)%Hko^9#5?VkJZ zx$WK`mfPuT>${!4?d-dczW)C2mfP>^pZQ{awS3-;+nHIg?1vejH)DHeY|qSh%l+X6 zFJ9sQWyaUd+6S|botHV=KR0D{Esb+~mhGFf59jQMxohjj^7Zq!W8UY^Ynu1HnAbaR zTjp)+g44O{*T@FvHfyvn~#0HUkNYA zw&~c<*74x=QpT~}dm`mT`iadvvFlHK%88pFxcPy*IE4=s(@Z#*#i?d5Ff84!JxhW@@UYtyNaU$u(38WV%a9*68 zd2v$a#mSf#CsAIUM0s(d#lh%{1JD--k}p4HI!Ju^Y4`s8qx0$@ z;`O)f&g++l&g%_Hn+j;fqe|6IJb@H+E>e2r?zALuw_i6@cpScND&JhNY|W;BZ%tTkZR_)G_q1*A7`1)f z_W2&8uiN&}w!OZsZpWV8akD$NY{wqmaW}iRVAtpE-C6ejzPI=Htl{T+`Frm9?@iP6 zo!0}|hrXSMzTnUec|gA&FLz!~#yYR3Hhtg=&b)o*4u(E9^52m=zpz^_bX?eoOP}wh zi9a8~ua~}^mrs`6aODoKd>gN9#FdS>vSBXcyk5^(zUJEPc+|aq_xAVu&a1!uuQ!9v z>#fk+pPkn`JLt~m-r0S3?%TQc*Lz=cFY?~!-`oBN+y3C&=H>lfAKdSQo#ov7>y!I& z8R+%N`=4x!^W(41XTLhH`|6zR>z{VCzooCvO};uG`Rd%`tMiJl&e6R(C->^1^Xh!t ztMg2+&ZoTo99jQ#-hSJ%cC9};Z(q9B$og(QSwA{&{;<9|7<==)>TTTHuCKiLgZlPu z+PdnzO>B4GoWOXSobSB(QGfHJ{x)@H4K24d^9l^osA$O zI&aQ4ym=_RIlJ)YtioH*zdZ!r`fkS`rni|tJ8!dJEnh$DzGr*ZiFMm~o3s7?_`J<~ zzaRa#1)sBEyZrdSE!=e87H#jMk1eiu-j*~j`PkBF=WW^7EPLPbo^@>5?q%)EZgct0 zvJaMR_p<%8Vtc)8{o9IZj%b3^ZjeYa`G`^JUO&!7J{e`(%Mw4T_mlU2*Mow$z^+kRqOPTl9}jO8{@=PcWD`n%<82X5D2 zrMH3mIuhau6)k5&%d@$uigK5x92a; z+l}qNv7NW}*RA*6`R|?Wy0b6tbb1`TdG7gk?`!XE^S$PK-~R{u^1(iM@Oh7J=h4?b z`My5c4%eREJU4xFz2?nx%{SLy-kkq^bB_1Tx!O19Pv3snF6T|(oF9F2KJ?9b&NqJ& z-v04@abEA;Ij8sEzFB>1(Yourdk+5Y?9Ka^YwNr9kM**^X~fP`&S>E zT(^#_Gizx1_@vvJ{JZl$<@TM-dv_A=eah!~T)a>1b>5wIdw16D-IKm|XWHKV?7w&2 zc6Y(rv2HB4(f!eR@10nuo%g=?_uWtbvGYD-TW6Lm|DJV!Jm~wJ?ehff-4nF;xovA; zxergw-sgRM!N(WYI`2#Fd&&Ej-2RenTwd+GuN-&Y*JdpH#FMG_b+^CncGq=pc>l({ z_FZcINZ*LuT-nVBh{X5>jqh;4^?7FSJgUrQOfiGA#B;rprmb81@$wr$|+&)nY7Z4K@7 zq3!gu_&)NnkjUT==_d^zY7pfBbf2{oeWT`}f1y!jCVv z*593vu{mqOTJC%}!SdlOz=z}7kMSR!57)v!zHW6s{2u;raP#3h^oIkK57&r4CMT@E zTg!hy0rXzecQU~ zd~A51pNEf)2g_}3xSdV+vuWEm-R`E(-SqxVx3x8H9a=uNb!PeamXB}y_qMj3Da&^G z1OMR<{Kt;Z-O;`0KK$PM*xR??*3n$4^P-X{C)p8 z@-?2Qe;m1;qif5y9eclL-yeQ%K2GhI(}8u-`525@_Tk{4&d0fZ<~hK}$Zd{%z3bv1 z7blk6xUiijI6f{nE&sl;UvAbqAO3cK+}ZyRZtu~4efqQW;o9_vYp5Td-hQ}-`SGXQ za;@^?Puu9);fL#jAFd02_?!IUn%l=;zTQ*Y57*f~Tu1xxH~PcV+mF9(tLtMQepWtQ zoBD7q>ce%X57%%$T*LWrJ?6u8m=D)snuDIUIo)a7gHGGI?zG+6PP=&Ow97^7)bjrC zzgb_bF>BoVYJIaNtVwIiavR^fR?q5NGnU)(Jf?l0x7^QnxA}d^TDDfKRcp;!w>GRz zYs=cUcC1}%&$6B053ED$$U3%8EZg_}bNkM$b8BdgtPAVXy0SEV|8Ct_x7M9?Z#`I# zmR3hL?fW0rpVnX2-_{T7AM0nQ`JS~K{Wtn=^xx>e(eLr!ZuI*d&~EhK=)ci_qyI*~ z=WXpq|Be0|{Wtn=^xx?BJEPs`_p{J$^xx>e(SM`=M*ofe8~r!>{rzh<`WO8{kQsW_225h)$e&}yVZZI|5pF4en*4tR{yR3Tm85C z{q1hI`fv5$>c7>0tN&L2t^Qm6xB74O`(4^@_225h)$jLeyVZZI|5pF4{#*UG`fv67 zo!f5p`@P$4_225h)qku1R{yR3Tm85CZ}s2mztw-M|5pF4{#*TyMcS?YTm7DAG|#u2 zquSA%zO`A)mj@ApT$)9<-Y^W3N1>A%x| zr~gjBb8+oX|DFCj{dfBB^xx^f(|@P`PXC>L&zstv{yY75`tS7n-PG>%-|6?9s@>~1 z*VFFx-|N5Ef3N>uzrShiUjM!Rd;RzN@Acp7zt`{iSi9GMum4{Ez5aXsj$4`;p?0s| zS%Y@3-`~b&W~iANYK~{xz5aXs_xe4jYxnx^^_wSZ_xkVk-|N5Ef3N>u|GoZu{rCDk z2WBo&V;lF{a%i;J?MYX|DfM_()OVLLH~pP2mPL_ zwg>$W`XBT^=zq}f`eu93?|E$V+@U?_f6(vmNAtfkXpZ%oW4-2#OMB4&px>F7_MqSM zjP{`aLH~pP2mKHFAM|_v(HtAL2mKHF&1f}eYubbU2mKHFAN4=#f7Jh||53kV$@ZxK zQU9aVMS#sQ*#_qy9(z zkNQ1tZ;$#P^*`!=)c>geQU9ai1l%J?eL?&>r3`Dyr2k3(ll~|DPx>88v?u*f`k(ZB4%eRa zKk0wc|D^v(|C9bF{ZIOz^gro;(*LCYN&l06$K}o0v1UZG8PRM;G~1K@C;iTtHD}D) zll~|D&YU%8&f1gyC;df(Px_zqJN9W$`k(Ya>3`Di zTtM?2q&?|>((i0sb2hF$>31%_IhWrYFE!`$n_0bPRo=>{JRfSm z>o=>{%<46>dd;j}GppCk>NVF8npwT(jA1jY*UaiQvwF>}UUOZcnbm8~Z8Wob&8%K? zouQf4Yi9MDS-s|(Lv!rcTz6=$J2bO;&8%KCtJj=?Y-aVE>k!SXUUPid%<46>dd;j} zbFA3R>NT@^&8%KCtJloxHM4rntX}&=zgfLzR&BLq2H`t zGppCk>NT@^%{iTBRNT@^&8%KCtJj=fZD#eFQ>@Lb zUNfuLJQr=w3pKNP%{7>2R}UNfuLT%T!X^_p3|W>&A6 z)oW(;npwSORi<)}b5QM1{eSBB9Jsm8)6DQSGkon& z{jT{m&x@N=z|9O_GsD-+@U=hnJ8o*Ok2N!V&2^z>hOaqI+|2MbGkneI;%0`gnc-_r z8#gn2&9$^XBG%?w}jJiM9VYi9VG8NTK`Su?}e zoPKVO*P3%?%{8dzxq93XsGwau!du+}62ne}UC{hC?7X4bEn^=r=cG_!uqadk86*Ub7g z$4t$vUo-31oCj)V{hC?7X4bEn^=po&npwYQ*0250|3m)|{bv4}+YkL_1Do?)&3Uf&L%$ip=Kqb_e(3+9--Xj=Ua*-LY~}@7==I(j03x$C}N&U^6e+%nLU2g6$vu<^`McXwAG}bBU#y7i{JQoAYYT zd5z}yw3!!d&TllwsqOPP)x2OcFWAfrHuHkbykIjg*vtzy^McJKnr2?GnHOy41)Iw? z%|+g3Ua&bw(##7s^McL1U^6e+oHJ?8nKUzl&CFmkGuX@wHW!7PxxwcAUi(MC3&icG z{-65I5;m8Bnpwj3Q@``y%{*ZNi)|TqbHi_5ajw#;}<&Y-SAGPyIjj z|I}~Zu(_1f%o{fIhRwWT`>Fq@{-64P>i?KlPhWY(MpzPi!tS!>Rkw-&5LYsp%+R;*QP z&04oMtW9gn+O~GAU2D(Uw+^gB>&QB`POMXFV4Yd#*3cSR7uKb9WnEj}tsCpsy0h-B z2kX&#vVOPzu>Q3Evi`PySpQi1zv%y>|BL=F`n{r0zUX(JH(&IB(f>ui^KLPI7ZZN@ zqTiHXzUVjUmoNHF`{j#%6My-l-_&2e=r{S7FZxaY<%@n3fcc`|6kxvScP=?!^qU6E z7yV!Kn+nVq{U!tRMZfcX`J&%-mweIxMgJH5&OygH=$IOe^MNrr7?Xo>xiY2)uD2^qW)3nEo;St`Eh0V#f53=^xW?RxxAx$MlcsAJadk-w0yN z8fHxYnEo;Sh6rPbFa`!QrhiQTnEo;S#_lqv-#Pb~rHg^N7?_KhxR{BH39Fd5iZQqt zgNu>3n8}Kfw-|ZLxPF6f8P`9q-=JFzx@BCyOTBSTD@NO5rY(lqGOmAIzjLh_*Kcqw zX4PW4D<;x1uHU3tjF4qqze~k&O)Q2^V$v(dxnhVb&eO)oRg6=`rR12Fin*ScmWs)! zn2d@cm6(T$xu=+iiXo>Ma*E-ujO#bi6mv{*zBlHWVvZ@}`p5N;>o?C7^Gq?%6!T0m z&lK}aac($fnli4Rxe9X?<|?jPgt^LB{mfOEtGL7+<|<$HGgo1*!d!*93Ud|aD$X;9 zxyo1l+*P=%eAWL||5yEA^>bKpE;=k$Sgf#E`Kq7CitB~>s-MToSN&i0f7Q=s<*WX$ z`VH%Z(F&s#Mk|a~%s}O<{;&F3t$fx0RsUE0+*ZEo|EmA1{;&GK>i??$tA5vJV)!S9 zfAUTLH~ruAyPO}_F7r)4^A(r&HntRW&Svq9+&#_P5(Fj-}Hae|4qLEqI}c; zO+PCZRxGSozUlv_|C|1A`oHP_rr)?g7_u;AVaQ_mD29*1ki`H}j1t5EQk=VwA*3*5 zF^Ck`Rl|^lAqztmhAa$O7_u;A`KI5nQoiZ`rr$V1zUlv_|C|1A`oHNn%#;cJ2AVRV zpFInEmI?h6`WduL=%3I(p?^ZZ!KWB)$b^354KcVLgHSQJ9!@O=qGBK_2BI>d-!+FA zXpe#RxE7oV{VZE1^iSw#+A^VkLcdXoOz5A`KcRm@|Ac<#ErzOMs4CoBCiEMr2>%xT zEk-NCzs0qU7`n=YeuGz;&_AJnLjQ#R3H=lLC-hI~pVU98e^URX{z?6l`duFhLl=fF zhPN`Qe^S3|CNYp7*R$i=NnF#8>nEAi&)8*B|D^s&{k&aV{|j@MN&S=h4S{z?6l`X}|fZkS2^t{rAl|D=A`5HqQNQvalWjxR<^;<{oc^&2dU z@sdpHpVU98e^URXe%Bo{see-cq<$kPaUC*d31bi~u197{|CD}1X)&G>&>&O#r}Vp4 znJN8K`ls}}o*w`q079npPwAi1Kc(M5TU^)7l>RCGu5V^azah6w>36L&Q~Ia$PwAi1 z4>4p)KgbZXjWH(>1934c5VHcA(m$o&s7$8xPwAi1Kc#<4zw1~rPZ0A2nbPl?R;KhD zsEhf6OzEG}Kc(NSL8kQ+5Si9Lt)GO*w0?tkF^7<8{ZvGz^&82_w0;A5nbtq8-<(3G z^-t?Ju9Io~)B30NPwSu7Z;m0;`pq(ATECH=OzWT4KdpaSzZr*2>z~#?t$$kowEk)R z)B30NPwSu7PiDk5;fx)%s>L)oua-^%jtDoqIp~H0by9S-E{;vM6{;vM6e%Gbb)!)_M)o(^4UHx7C zUHx7CUHt|Z)79@9cDnkz`n&qO`n&oKHKwb-tG}zitG}zitG}zitKaNL$dZsHF$0pW ze%Hg()$f{k3`C}@zpKBizpKBizpKBizpLMM^K|uh_4oAm^!N1l^!N1l^!M}|tV~b8 zv8(j-_w@Jl_w@Jl_w@Jl_w@Jl_w*afN>6`JziX)J>F??9>F??9>F?<`sF|LAGA3k9 z$e55ZUa`~C@0xyk`g{6&`iYqkGa+U|%%rEkr@yDar@yDar@yD)@Mn7Zd-{9&4T8oX zXnOk1%cQ5@*jRe{`})ler?0=S-}T-Y7>$9^7%NL(e_wxJe_wxJe_wxJe_wxJe_wxJ ze_wxJzd_UV^&2)#Uw>b}q0{vB_w}1qNMCbJUw>bJUw>bJUw>bJU%wgX^!1yANMFA}*Yx%G^&5CiU%yeg^z|El&5V8nu$j?6 zqu(HG48mqc|BU__{f1)W8hB>(&*-1gKcjy}zmdBbn2ni>n7N3-*%-mgjQ$z@Gy2VB z$1rVX^c$$njQ$z@Gx}%r&*-1gZ(bua`e*dd=%3MVlrIKwGoyb-|BU__{r-2;nbB`B zH#7QY^c&F4jDE9FnbAL^e@6d|ezQ?A5||nN26!{0-wVb zQDs)YS*pzHH&2yW{j>UK_0Q^`)jz9$R{yO2S^cy6XZ6qOpVdFBe^&pj{#pIA`e*ge z>NlR4S^cy6XZ6qOH?x&l{j>UK^&4Nzto~X3v--`nWLE#I{#pIA`ptM{R{yO2S^Wl+ zGpm1A|E&I5{j>UK^_z#uto~X3bNc7>&*?YF9AlC(CYd??bNUT7$LvhZ&SXx%`I*e= zpVL35e@_3L{yF_~`sehUzt5chIsJ3`&EjWH|D1laH<{CK9xVo?GpBz}|D1k9(=nbI zgVUMQZ$>9``seh|={HQBIsJ3`=k%N3$((*OJekvPz&dmKz4m+N^v~&^({FYobNc7> z&*`7jKc|0A|GfTr{qy?g_0Q{{*KbxX^ZE^I$IM*j_45Ro*Kc?`^ZMuY&+DJpKd*mY z|Ga*4BAM4euYX?uy#9Ip^ZJdR#?W`>_0Q{{*FUe{OkU>o&+DJpKd*mY|GfTr{qy?g z_0Q{{*FUd+UjMxQdHwVH=k?F)pVvRHe_p@g@)$19y#9Ip^ZLy@WnTZh{ssLD`WN&s z=wHykpnpNX8Nn>*Hyf1&{R{dR^e^aN(7&L6LH~k&qqJGjzo36X|APJn{R{dR^e^aN z&~N523;GxIFX&&;zo36X|APJn{R{dR^e^Z)3?DO!Ssv|Dt}gkXh8fsDDwvxyUT) zU(|1aH;eig^)KpQ)W4{IQU9X;Mg5EV7xgddH&2;G{YHzksDDxaqJ9JZS=7I%e^I|- z|19cX)W4{IQU9X;Mg5EV7xgddU(~;-e^LLU{zd(Z`WN*t>R;5qsDDxaqW(qw+)ftt zvpZST4;5rl|DygS{Y(0n^e^c*`kW>GOZu1eFX>;>ZzeQL`j_-C>0i>nq<=~OlKv(A zOZu1eFX>;>zodUj|C0VC{Y(1Im1ar*lKv(AOZu1eFX>;>zodUjzuD6)>0i>nq<=~O zlKv(AOZv^}Wl8^%e)D=+(r;!jOZu1eLlarjzodUj|C0VC{bu^Iq<=~OlKy4=%lZl2 zEbCv^zpUTfYnJsd>tEKttbbYmvi@cL%len~FY8~{zpQ^*|FV8ID&_{Wtl#Wxmh~^| zU)H~@e_8*selxXM*1xQOS^u(rGqzdQzpQ^*zj@m%>tEK-F=bi*vi@cL%lggYW?8>^ z+?dCWk@YO=U)FCnH_Q5$^)Kr;r<-N{%lgghW?BD={uTWz`d9R?=wH#lqJKsIivAV- zEBaUTujpUVzoLId|BC(<{VV!c^snd#RAOX4EBaUTn+wj0{uTWz`dO^3=wH#lqMysk zivAV-EBeh4XGQ;t{uTWz`d9R?=r@L+75yvvSM;ywU(vs!e?|X_{uTWz`pxQPMgNNa z75yvvSM;ywU(vs!-z;BN^{?t*)eqHVRsX7fqyAadzp8&#znQR;8rs()4gs{U2| zW}UODe^vjg{#E^$Kvwmm0$J6+s()2Ktdmv!tNK^&)e@*|I{x$t;`q%WY>0i^orhiTUn*KHYYx>vp zujyaYzovgp|C;_a{cHNoi3gj=n*KHYYx>vpujyaYzovgp|C;_a{cHNy^snh()4!&F zP5+wyHT`S)*YvOHU(>&)e@*|I{x$t;`q%WY>tENuu76$sy8dtENuuHS5X*7chS4(E||{ptENuu76#>`Q>cr-_UO^KO6ct^l#|j(7&O7LqD*R4gKa@v!UONYc}+6=r{WubC%iA zzoCCa|Azhz{Tuo>^l#|j(7&PIoOCwyZ|L99zoCCa|Azhz{Tuo>^l#`lPaSA2W~#HH z-&}Pz^l#|j(7&O7L;r^U4gDMX!J2I7=L@o--yCi>^l#|j&~GL;+(9<6Cm)W4}8;>o7|P5qntH}!Ao-_*aUe^dXa z{!RU`Pd4>$>fhAAsee=drv6R+oBB8PZ|dLFzp39Gc{cTL>fhAAsh^w3rv6R+oBB8P zZ|Vn$vZ>#^am*XXym8DMXH);C{!RUx`nlI|7unLkrGHERmi{gMTl!(7Z0X<9zomam z|CatO{agCC^l$0k(!ZsDOaGSsE&W^ixAbr6-_pOOe@p+C{w@7m`nU9N>EF`7rQa-c z%tB{N|CatO{TxWlMrTX^mi{gMTl%;3^C8*N4@_lC|CatO{me+V^qZ&7mVS^bTl%;3 zZ|UFCzomam|CWBpDvU|C^>6Fn*1xTPTmQCx04o4iw)Jo8-`2ma-%NkD^>6Fn*1xTP zTmQEHZT(xa6st>5f-w)Jo8-_~z#Iov?D^>6Fn z*1xTPTmQEHZT-+@w)LAC&$j+;{oDGt^>6Fn*1xTPTmQEHZT;K&xAkx9-_gILe@Fk0 z{vG{0`gip2=r@O+9sN7{cl7V*-_Z}CWk>&xesk*C(GR3$NB@p~bL-jBzoUOg|Bn70 z{X6=1^qXnVj{Y6}JNnJHXGi~z{vG{0`gip2=r{MC9sT@7cJ%M)-_dWTJ3IP!^zZ23 z(Z8dANB@rg9sN7{nTzb`hvfXfU@ZMR`gip2>gU(9tAAJju6~d%j7E0#@9N*xzpI}+ z%C7!h{btFttAAJjuKr#9yZU$a@9N*x594K5|E~UB{k!^i_3!H6)xWEMSO2d5UH!ZI zclGb;-__4JWmo^M{$2gM`gis3>fhDBtAAJjuKr#9yZU$a@9N*xzpH;&|E_-CB)j@| z_3!H6)xWEMPye3&J^g$7_w?`S-_yURe^39O{yqKdLH6|T>4$`~r+-iXp8h@kd;0hE z@9E#uzo&ms|DOIm{oF$K^zZ54)4!*mV+aQmjw^fm_w?`S-_yURe^39O{yqJB`uFtj z>EF}8r+-iXp8h@kEJVQN?CIarzo&ms|DOIm{d@ZN^zZ54)4!*GUq3sMef|6T_w{oW z+1J0Xe_#K;{(b#yMfUZB(b?C(uYX_vzW#mv`}+6w@9W>!4`7DH$iDu4{rmd&_3!K7 z*U$ZAU;n=Tef|6T_x11V-`Bsde_ub#k$wI9`uFwk>)+S!H5an4AJPt_oqhfL`uFwk z>)+SEuYX_vzJ4Ag`}+6w@9W>!&mHAJ|AGDk{RjFF^dIOy(0`!+K>vaM1O1Fr4)pVO zInaNg|3E)engjg@`VaIU=s(bZp#MNWJe~vn2l@~6b16B{&pYKn|ABt)DF^xw^dIOy z(0`!+K>vaM1N{g35A+}CKhS@m|3LqN{sa97`VaIU=s(bZp#MPsf&K&i2l@~6ALu{O zf2f~@&7uB7{fGLw*c|FV)PJb|Q2(L+L;ZYV4)t@hIn;ls|4{#-{zLtT`VaLV>Oa(f zsQ*y^q5ebthx!loAL>8Uf2jXZ|Dpav{fGJw^&jd#)X(nZQ2(L+L;Z*P5A`4FKh%Gy z|4{#-ejq=G`VaLV>Oa(fsQ*y^q5ebthx!loAL>8Uf299NKW~;J{YUzb^dIRz(to7? zNIy@RBmGDEkMtkuKhl4s|49Fl{v-WI`j7M<>1Wq+r2k0&k$wg#NBWQSAL-|ja-^Sa z3zw85{YUzb^dIRz(to7?NdJ-kBmGDEkMtku=a}LJ2Xds}iw@*S|B?P9{YUz}2tk;+ z9O>ug!h(i-%8~vf{YUzb^&ji^Vg)(Yf2{vlzZWjZvHoNI$NG=;AL~EXf2{vl|FM1+ zFUR_i^&ji!sN=;Ba;*PY|FQmK{a*Ya$NIehLXPzx>p#|itp8ZQ7e&ai{$u^e`j7Q{ zk%TZ>VYI@q=2-u+{$u^e`j7P=>p#|itp8a5vHoNI$NG=;AL~EX&%fqaKLeX%{m1%` z^`Gcx3B!KpME{BY6a6RpPxPPYKhb}p|3v?Z{uBKt`cL#TwDGD9InjTj->Wy|L_b%X z6a6RpPxPPYKhb}ppSR74{uBKt`cL$G#fO~eXBxxd=0yLA{uBKt`cL$q=s(eaqW?ty ziT)G)C;CtHd&P*H=s(eaqW?tyiT)G)C;CtHpXfi)&rOE)&8dFgH>dhf^`GkZ3T$C2 zbE^MT|Ec~{{fuQ!^`GizgTn^rRR5{|Q~jsOa+g zs{d5~ss2;_?0ru4pXxu=&*SA(|Ec~{{ipi*yqxN1l5?v6RR5{|Q~jsi#tigxvN_Z5^$v2T z|4jdx{xkh&`p@*A=|9tdrvFU;nf^2VXZp|dpXoo-@AVRLrvFU;nf^2VXZp|dpXoo- z&y43x|C#p$0juK!&Bx&Cwg=laj}pX)!@f3E*r|GEBi{pb46^`GlM*MF}6T>rWLbN%P~ z&-I_{_bM+r*MF}6TtA1JbN%P~&-I_{Ki7Y*|6KpM{&W2de$Mru>p$0juK!#=m!EU} z=laj}pX)!@Kh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_e zKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_e zKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKh!_eKhi(aKhi(aKhi(aKhi(aKhi(a zKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(a zKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(aKhi(a&$-9zFJz>Dq<^G; zq@R7yNdJX?20j=1FZ6qjlU(S((0`%ds|@8rzt;xOg?_IOo(uh6BRm)SFZ5sN=c03= z|3d$T{tNvV`Y-f*y|Y~CztDf7|3d$T{tNwH7b0FyB^UZH^k3-bt8=0MLcdoy%7y+5 z{TKQ#^k3+|(9dA!LjQ$+7CTc7-~ssB>{rT$C(m-;XDU+TZq@8$X9<@s}||5E>@{!9Ir`kCrn>i4=ZxzvBD|5Cr# z709K2Mm(4LFZEyQztn%J|5E>@ey=5yOZ}JnFZJ`;;j?q8|5E>@{!9Ir`Y-ig>SxY# zssB>{rT$C(m-;XDU+VXoG`ZC8b!l>`|5E>@{!9Ir`Y-i+&4XO&ztVrD|4P4CkII$) zEB#mcuk?Fegk0&r(toA@O260h$d&#p{a5<0^!r~I24EB#mcuk>H(ztZn@ z?{cNzYvtrh|CRnL{a5<0^k3<}(toA@O8=GqEB#mcuk>H(_ZmC7(toA@O8=F9uep;e z{a5<0^k3<}(toAjYw^Tu@#ISXmHsRJSNgpkNv`!@>-TC|xz>NJ-)ofQTK~2FYyH>y zy^c?=^yy~a{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5 zfB66K|Nj5NmVW*}{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q% z{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@% zAO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk z{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$j zKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8 z{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5 zfB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG z`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A z|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW z@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K z|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q% z{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@% zAO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk z{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$j zKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8 z{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5 zfB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG z`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A z|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW z@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K z|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm333|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn z@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX z{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=U zfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%o zU;KaZ|Hc0o|6lxn@&CpD7yn=Ue~szq|BL@G{=fMD;{R(*KmT9+fARmt{}=yX{D1NP z#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU*r1u|Kk6P|1bW( z`2QN$&;J+yU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=U zfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%o zU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD z7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP z#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn z@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX z{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=U zfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%o zU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD z7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP z#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn z@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX z{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s61Ve^-B3e^-B3 ze^-B3e^-B3e^-B3e^-B3e^-B3e^-B3e^-B3e^-B3e^-B3e^-B3e^-B3e^-B3e^-B3 ze^)>MU;KaZ|Hc0o|6lxn@&CpDS66>me^-B3KmT9+e|7bD^>_7m^>_7m^>_7m^>_7m z^>_7m^>_7m^>_7m^>_7m_4EJ5{}=yX{D1NP#s3%oU;Kad^!N1l^!N1h|Hc1TPe1=( zJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elX zJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXJ^elXef@p?ef@p? zef@p?ef@p?ef|7@@&CpDS6_c$e_wxJKmT9+fARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD*Npxd z{WJP!^v~#@(LbYqM*ocd8U6f!@&7fWe@6d|{u%xJfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=U zfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%o zU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD z7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP z#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn z@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX z{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=U zfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%o zU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD z7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP z#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn z@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX z{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=U zfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%o zU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD z7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP z#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn z@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX z{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=U zfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%o zU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD z7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP z#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn z@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX z{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o z|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt z{}=yX{D1NP#s3%oU;KaZ|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KaZ z|Hc0o|6lxn@&CpD7yn=UfARmt{}=yX{D1NP#s3%oU;KalpSJF&NAd$P>v;A$N526o z5f`Lj2_YebI1zHlA&QW~32|tT$GekwXPo)7VL7oIHX9{Xh=7uS#Im->Gq!uG2}v#x zLOAR@!3d<}`$+W?_|(`*7Kuaurn{%Qs-Ef^Pft~SKi2=J|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdTTZ2!mh>;KdLr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VHFYJF|zy3e{fBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9De{XO8TEYN?0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K; z2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx(5?Nq z_TSonYd;1c3_!Q`-`am`|E>ME_TSonYyYkNxAx!Ke{27({kQht+J9^Rt^K$5-`bA> z2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx$D+Wyz} zzqbFi{TP5S0AT>a0QB1a*Y>}*|F!+E?SF0mYx`f@|Jwf7_P@6Owf(Q{e{KJ3`(N9C zw*PGZ+5WTrXZz3gpY1=}f42W@|JnYt{b&2n_Mh!P+kdwIZ2#H*v;AlL&-S0~Kihw{ z|7`!+{|DFAJ_TSlmXaAl3clO`ee`o)l{de}? z*?(vMo&9(A-`Rg>|DFAJ_G19T0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K; z2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF0}Snc6xU zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c2GAecPXmYs z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ez&MKWPBb0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR54E0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R#buKhHC(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e7Yuh~xn7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy0QQ^q(*Q;T7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4Pd`z zKMi0sfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=Y`;Yb??LXR20~ifpG=R|ncC`O!|Iz-V{YU$c_8;v(+JChFX#dgvqy0zw zkM_6Fmvj1fN$^Mi5C;LzK zpX@)`f3p8%|H=N7{U`fR_MhxO*?+SCWdF(jll>?APxhbeKiN+M7!6=FfYAU(0~ifp zG=R|nMgtfPU?=-e_R|1H0~ifpG=QD#KiPk>|78Ek{*(PD`%m_t>_6Fmvj1fN$^Mi5 zC;LzK(*Q;T7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y{Tusf0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtuzzbm4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(Eu+0 z+F!N~pZvt-w>@R(@E@*+*AAZXark`)kHa_|9Xw*;@V)EdUtJHs?ciVU4&T2Xe#5~J z{SJR}J^ZPIp937OpA(2)e%Dh^4j*3+F8p^r`{eNQdU$m`e0n{6=HO+R4xe8Sw?A?D z!S{a8o0I+Ed;jkLdB+E?UC%vBKm+$!CdQN}& z!S$Se{g40RByZmNiANg<3|~3^07|g^1>?!E-ySBdwJn;ugeP$7CE>s zFFfqx@H-CvVS9PuZ-L7TA^7Em-0bo~zjS%=hrWL=ykz+D!fUNAFT8l>^1>@nE}wYF z{PKy%LNA|qaOm=hpH?oP_~U>1gi3JvLhS5Llc*me)hP$^cU9UrN0R-FO{>Gmy)#0%m3suUV6p+<)xQ>UtW5R%;l9Q zb}p|xOL2MS>4M8EkM&+&d2sRa%D?z7ul$qd^2*P!mse;D2UqRY+4;S?UbRb^0^0YE}#3?_~mmy5??+)yBFSi^7?Y? zS>DU7r*ST~9>2fbdI0Zo>!-n=`Y-qH{nLAY?dJ~fKmOK(!{vvk-#Oeode=MO|J>W{ z;l^(~`^mTd-v9jF@BG}s;r+)4r}NI2r}fVD+d-uzx>McgEKogx5L}_UVZJ}-~C0O|JLLC zH}5=u|K`cJ58u4`;OXJP&HZm2K6v=x{)6k)xp{PW_~_Y#uRnNlc=+yvN6#L<_wd1! zx8M3D=XU*KzkJJ8x_`KN=kWaS{^91y;oH|we)Dki!QstCA08ZT?jOE!{p9sbedxU} zudDO+{=-LyXAcfvKez-Rxs-#8IXpbP=gsHdzW3d)-TRwA`^|SAy!-CMM_+$>y^_A* zpE=1RXX83Owc)}2XKszBAKcu3@bvAs{*g2K*~2&AIXrmx@UGK+{mZNJ=1N^J{rc5! zF8RyralJ&R_-c*o`0V=qaDt}?XXqw#CC*>F_xFF{z4x9zc=puD2ZwJyxOw{giSu{$ z?>~O`!QuI%hkonvdvCsJkDomJ`ol*z?>l3^=?c94*89%y@C#PGcfF`*UtXi@Ir;oI zuQ$@u!}BNCEBxk)U61zvjE^Hy7!gzPh{me*F0N(>1!@D$du5{rBPUtB0F!e|eQY@Wk~`p%on z^6UTq8~Wyc`u4qla__JI?;rfZ|N7(e7vKFKU;NtdfAMR7^2K*v|DS*V-@f?nZ+`Ke zKmFo6zxTy=e)z?|`0iVO`+5bP^5CQgr#(2aw?8>J`N8RZ?l-;vr`~?g+Yb-E=-xlR zci=q!`S*Uu<1_w;5e&0~U;gZm{v`?N&wuMT{`|K-dHb!ubv8__ZTu=Gt S^WJyzgVP^;fwxZ%Z~tF?pN;VV literal 0 HcmV?d00001 diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.fixtures.tsv new file mode 100644 index 0000000000..b8f9733dba --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.fixtures.tsv @@ -0,0 +1,40 @@ + 0 + 0 + 0 +a 1 ▁a 266 0 1 ▁a +Hello world 7 ▁ 261 0 0 H 552 0 1 e 264 1 2 ll 412 2 4 o 274 4 5 ▁wor 427 5 9 ld 317 9 11 ▁Hello▁world + Hello world 7 ▁ 261 1 1 H 552 1 2 e 264 2 3 ll 412 3 5 o 274 5 6 ▁wor 427 6 12 ld 317 12 14 ▁Hello▁world +Hello world.\nSecond line\ttabbed 21 ▁ 261 0 0 H 552 0 1 e 264 1 2 ll 412 2 4 o 274 4 5 ▁wor 427 5 9 ld 317 9 11 . 262 11 12 ▁S 296 12 14 e 264 14 15 c 411 15 16 o 274 16 17 nd 284 17 19 ▁l 492 19 21 ine 415 21 24 ▁ 261 24 25 t 269 25 26 a 279 26 27 b 344 27 28 b 344 28 29 ed 267 29 31 ▁Hello▁world.▁Second▁line▁tabbed +The quick brown fox jumps over the lazy dog. 26 ▁Th 277 0 2 e 264 2 3 ▁qu 346 3 6 i 282 6 7 ck 307 7 9 ▁b 309 9 11 r 288 11 12 ow 306 12 14 n 268 14 15 ▁fo 359 15 18 x 421 18 19 ▁ 261 19 20 j 381 20 21 um 529 21 23 p 352 23 24 s 263 24 25 ▁o 513 25 27 ve 366 27 29 r 288 29 30 ▁the 265 30 34 ▁la 339 34 37 z 351 37 38 y 275 38 39 ▁do 464 39 42 g 356 42 43 . 262 43 44 ▁The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog. +tokenization and segmentation 4 ▁tokenization 461 0 12 ▁a 266 12 14 nd 284 14 16 ▁segmentation 335 16 29 ▁tokenization▁and▁segmentation +Antidisestablishmentarianism 15 ▁An 408 0 2 ti 523 2 4 d 273 4 5 is 278 5 7 est 323 7 10 a 279 10 11 b 344 11 12 lish 454 12 16 ment 508 16 20 ar 353 20 22 i 282 22 23 a 279 23 24 n 268 24 25 is 278 25 27 m 320 27 28 ▁Antidisestablishmentarianism +water running walked faster apple book work play 15 ▁water 312 0 5 ▁runn 467 5 10 ing 272 10 13 ▁walk 348 13 18 ed 267 18 20 ▁fast 358 20 25 er 270 25 27 ▁app 498 27 31 le 405 31 33 ▁b 309 33 35 o 274 35 36 o 274 36 37 k 354 37 38 ▁work 303 38 43 ▁play 313 43 48 ▁water▁running▁walked▁faster▁apple▁book▁work▁play +3.14159 x 42 = 1024? 21 ▁ 261 0 0 3 543 0 1 . 262 1 2 1 371 2 3 4 372 3 4 1 371 4 5 5 549 5 6 9 550 6 7 ▁ 261 7 8 x 421 8 9 ▁ 261 9 10 4 372 10 11 2 434 11 12 ▁ 261 12 13 <0x3D> 66 13 14 ▁ 261 14 15 1 371 15 16 0 548 16 17 2 434 17 18 4 372 18 19 ? 435 19 20 ▁3.14159▁x▁42▁=▁1024? +!!!???... 10 ▁ 261 0 0 ! 370 0 1 ! 370 1 2 ! 370 2 3 ? 435 3 4 ? 435 4 5 ? 435 5 6 . 262 6 7 . 262 7 8 . 262 8 9 ▁!!!???... +(parentheses) and [brackets] and {braces} 27 ▁ 261 0 0 ( 546 0 1 p 352 1 2 are 286 2 5 n 268 5 6 th 289 6 8 e 264 8 9 ses 414 9 12 ) 547 12 13 ▁a 266 13 15 nd 284 15 17 ▁ 261 17 18 <0x5B> 96 18 19 b 344 19 20 r 288 20 21 ack 327 21 24 e 264 24 25 ts 311 25 27 <0x5D> 98 27 28 ▁a 266 28 30 nd 284 30 32 ▁ 261 32 33 <0x7B> 128 33 34 b 344 34 35 ra 409 35 37 ces 478 37 40 <0x7D> 130 40 41 ▁(parentheses)▁and▁[brackets]▁and▁{braces} +café naïve fiancé résumé 19 ▁caf 484 0 3 é 557 3 4 ▁ 261 4 5 n 268 5 6 a 279 6 7 <0xC3> 200 7 7 <0xAF> 180 7 8 ve 366 8 10 ▁fi 518 10 13 a 279 13 14 n 268 14 15 c 411 15 16 é 557 16 17 ▁ 261 17 18 r 288 18 19 é 557 19 20 s 263 20 21 um 529 21 23 é 557 23 24 ▁café▁naïve▁fiancé▁résumé +financial fluid 13 ▁fi 518 0 1 n 268 1 2 a 279 2 3 n 268 3 4 c 411 4 5 i 282 5 6 al 281 6 8 ▁ 261 8 9 f 337 9 9 l 319 9 10 u 271 10 11 i 282 11 12 d 273 12 13 ▁financial▁fluid +① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 21 ▁ 261 0 0 1 371 0 1 ▁ 261 1 2 1 371 2 2 1 371 2 3 ▁ 261 3 4 <0xE4> 233 4 4 <0xBB> 192 4 4 <0xA4> 169 4 4 <0xE5> 234 4 4 <0x92> 151 4 4 <0x8C> 145 4 5 ▁ 261 5 6 <0x4B> 80 6 7 A 596 7 8 T 599 8 9 A 596 9 10 <0x4B> 80 10 11 A 596 11 12 N 542 12 13 A 596 13 14 ▁1▁11▁令和▁KATAKANA +ｶﾀｶﾅ half width 17 ▁ 261 0 0 <0xE3> 232 0 0 <0x82> 135 0 0 <0xAB> 176 0 1 タ 565 1 2 <0xE3> 232 2 2 <0x82> 135 2 2 <0xAB> 176 2 3 <0xE3> 232 3 3 <0x83> 136 3 3 <0x8A> 143 3 4 ▁h 512 4 6 al 281 6 8 f 337 8 9 ▁wi 314 9 12 d 273 12 13 th 289 13 15 ▁カタカナ▁half▁width +東京タワーへ行きました 18 ▁ 261 0 0 東 571 0 1 京 568 1 2 タ 565 2 3 ワ 580 3 4 ー 581 4 5 <0xE3> 232 5 5 <0x81> 134 5 5 <0xB8> 189 5 6 <0xE8> 237 6 6 <0xA1> 166 6 6 <0x8C> 145 6 7 <0xE3> 232 7 7 <0x81> 134 7 7 <0x8D> 146 7 8 ま 587 8 9 し 450 9 10 た 564 10 11 ▁東京タワーへ行きました +日本語とEnglish混在 18 ▁ 261 0 0 日 583 0 1 本 585 1 2 <0xE8> 237 2 2 <0xAA> 175 2 2 <0x9E> 163 2 3 <0xE3> 232 3 3 <0x81> 134 3 3 <0xA8> 173 3 4 E 595 4 5 ng 328 5 7 lish 454 7 11 <0xE6> 235 11 11 <0xB7> 188 11 11 <0xB7> 188 11 12 <0xE5> 234 12 12 <0x9C> 161 12 12 <0xA8> 173 12 13 ▁日本語とEnglish混在 +Привет мир 11 ▁ 261 0 0 П 559 0 1 р 382 1 2 и 373 2 3 в 437 3 4 е 438 4 5 т 561 5 6 ▁ 261 6 7 м 439 7 8 и 373 8 9 р 382 9 10 ▁Привет▁мир +안녕하세요 세계 19 ▁ 261 0 0 <0xEC> 241 0 0 <0x95> 154 0 0 <0x88> 141 0 1 <0xEB> 240 1 1 <0x85> 138 1 1 <0x95> 154 1 2 <0xED> 242 2 2 <0x95> 154 2 2 <0x98> 157 2 3 세 432 3 4 <0xEC> 241 4 4 <0x9A> 159 4 4 <0x94> 153 4 5 ▁ 261 5 6 세 432 6 7 <0xEA> 239 7 7 <0xB3> 184 7 7 <0x84> 137 7 8 ▁안녕하세요▁세계 +你好，世界！ 7 ▁ 261 0 0 你 569 0 1 好 570 1 2 , 280 2 3 世 566 3 4 界 572 4 5 ! 370 5 6 ▁你好,世界! +I love 🍕 pizza 12 ▁I 329 0 1 ▁lo 305 1 4 ve 366 4 6 ▁ 261 6 7 <0xF0> 245 7 7 <0x9F> 164 7 7 <0x8D> 146 7 7 <0x95> 154 7 9 ▁p 486 9 11 i 282 11 12 z 351 12 13 za 540 13 15 ▁I▁love▁🍕▁pizza +flags 🇩🇪 🇺🇸 end 26 ▁ 261 0 0 f 337 0 1 l 319 1 2 a 279 2 3 g 356 3 4 s 263 4 5 ▁ 261 5 6 <0xF0> 245 6 6 <0x9F> 164 6 6 <0x87> 140 6 6 <0xA9> 174 6 8 <0xF0> 245 8 8 <0x9F> 164 8 8 <0x87> 140 8 8 <0xAA> 175 8 10 ▁ 261 10 11 <0xF0> 245 11 11 <0x9F> 164 11 11 <0x87> 140 11 11 <0xBA> 191 11 13 <0xF0> 245 13 13 <0x9F> 164 13 13 <0x87> 140 13 13 <0xB8> 189 13 15 ▁en 318 15 18 d 273 18 19 ▁flags▁🇩🇪▁🇺🇸▁end +family 👩‍👩‍👧‍👦 emoji 34 ▁famil 446 0 5 y 275 5 6 ▁ 261 6 7 <0xF0> 245 7 7 <0x9F> 164 7 7 <0x91> 150 7 7 <0xA9> 174 7 9 <0xE2> 231 9 9 <0x80> 133 9 9 <0x8D> 146 9 10 <0xF0> 245 10 10 <0x9F> 164 10 10 <0x91> 150 10 10 <0xA9> 174 10 12 <0xE2> 231 12 12 <0x80> 133 12 12 <0x8D> 146 12 13 <0xF0> 245 13 13 <0x9F> 164 13 13 <0x91> 150 13 13 <0xA7> 172 13 15 <0xE2> 231 15 15 <0x80> 133 15 15 <0x8D> 146 15 16 <0xF0> 245 16 16 <0x9F> 164 16 16 <0x91> 150 16 16 <0xA6> 171 16 18 ▁ 261 18 19 e 264 19 20 m 320 20 21 o 274 21 22 j 381 22 23 i 282 23 24 ▁family▁👩‍👩‍👧‍👦▁emoji +zero​width and non breaking 16 ▁ 261 0 0 z 351 0 1 er 270 1 3 o 274 3 4 ▁wi 314 4 7 d 273 7 8 th 289 8 10 ▁a 266 10 12 nd 284 12 14 ▁no 489 14 17 n 268 17 18 ▁b 309 18 20 re 349 20 22 a 279 22 23 k 354 23 24 ing 272 24 27 ▁zero▁width▁and▁non▁breaking +quotes “fancy” and ‘single’ — dash 33 ▁quote 456 0 5 s 263 5 6 ▁ 261 6 7 <0xE2> 231 7 7 <0x80> 133 7 7 <0x9C> 161 7 8 f 337 8 9 a 279 9 10 n 268 10 11 c 411 11 12 y 275 12 13 <0xE2> 231 13 13 <0x80> 133 13 13 <0x9D> 162 13 14 ▁a 266 14 16 nd 284 16 18 ▁ 261 18 19 <0xE2> 231 19 19 <0x80> 133 19 19 <0x98> 157 19 20 s 263 20 21 ing 272 21 24 le 405 24 26 <0xE2> 231 26 26 <0x80> 133 26 26 <0x99> 158 26 27 ▁ 261 27 28 <0xE2> 231 28 28 <0x80> 133 28 28 <0x94> 153 28 29 ▁d 368 29 31 as 420 31 33 h 308 33 34 ▁quotes▁“fancy”▁and▁‘single’▁—▁dash + the [URL] token 8 ▁ 261 0 0 3 0 6 ▁the 265 6 10 ▁ 261 10 11 [URL] 4 11 16 ▁to 304 16 19 k 354 19 20 en 350 20 22 ▁▁the▁[URL]▁token +a b[URL]c 6 ▁a 266 0 1 ▁ 261 1 2 3 2 8 b 344 8 9 [URL] 4 9 14 c 411 14 15 ▁a▁b[URL]c +control tokens inline 20 ▁co 326 0 2 n 268 2 3 tro 429 3 6 l 319 6 7 ▁ 261 7 8 <0x3C> 65 8 9 s 263 9 10 <0x3E> 67 10 11 ▁to 304 11 14 k 354 14 15 en 350 15 17 s 263 17 18 ▁ 261 18 19 <0x3C> 65 19 20 <0x2F> 52 20 21 s 263 21 22 <0x3E> 67 22 23 ▁in 276 23 26 l 319 26 27 ine 415 27 30 ▁control▁▁tokens▁▁inline +https://example.com/path?q=1&x=2 27 ▁h 512 0 1 t 269 1 2 t 269 2 3 p 352 3 4 s 263 4 5 <0x3A> 63 5 6 <0x2F> 52 6 7 <0x2F> 52 7 8 e 264 8 9 x 421 9 10 a 279 10 11 mple 485 11 15 . 262 15 16 c 411 16 17 om 475 17 19 <0x2F> 52 19 20 p 352 20 21 a 279 21 22 th 289 22 24 ? 435 24 25 q 598 25 26 <0x3D> 66 26 27 1 371 27 28 <0x26> 43 28 29 x 421 29 30 <0x3D> 66 30 31 2 434 31 32 ▁https://example.com/path?q=1&x=2 +UPPER lower MiXeD case 17 ▁ 261 0 0 U 593 0 1 P 425 1 2 P 425 2 3 E 595 3 4 R 544 4 5 ▁lo 305 5 8 w 419 8 9 er 270 9 11 ▁M 442 11 13 i 282 13 14 <0x58> 93 14 15 e 264 15 16 D 589 16 17 ▁c 338 17 19 as 420 19 21 e 264 21 22 ▁UPPER▁lower▁MiXeD▁case +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 266 0 1 a 279 1 2 a 279 2 3 a 279 3 4 a 279 4 5 a 279 5 6 a 279 6 7 a 279 7 8 a 279 8 9 a 279 9 10 a 279 10 11 a 279 11 12 a 279 12 13 a 279 13 14 a 279 14 15 a 279 15 16 a 279 16 17 a 279 17 18 a 279 18 19 a 279 19 20 a 279 20 21 a 279 21 22 a 279 22 23 a 279 23 24 a 279 24 25 a 279 25 26 a 279 26 27 a 279 27 28 a 279 28 29 a 279 29 30 a 279 30 31 a 279 31 32 a 279 32 33 a 279 33 34 a 279 34 35 a 279 35 36 a 279 36 37 a 279 37 38 a 279 38 39 a 279 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Ω≈ç√∫˜µ≤ 22 ▁ 261 0 0 <0xCE> 211 0 0 <0xA9> 174 0 1 <0xE2> 231 1 1 <0x89> 142 1 1 <0x88> 141 1 2 <0xC3> 200 2 2 <0xA7> 172 2 3 <0xE2> 231 3 3 <0x88> 141 3 3 <0x9A> 159 3 4 <0xE2> 231 4 4 <0x88> 141 4 4 <0xAB> 176 4 5 ▁ 261 5 5 <0xCC> 209 5 5 <0x83> 136 5 6 <0xCE> 211 6 6 <0xBC> 193 6 7 <0xE2> 231 7 7 <0x89> 142 7 7 <0xA4> 169 7 8 ▁Ω≈ç√∫▁̃μ≤ +مرحبا بالعالم 26 ▁ 261 0 0 <0xD9> 222 0 0 <0x85> 138 0 1 <0xD8> 221 1 1 <0xB1> 182 1 2 <0xD8> 221 2 2 <0xAD> 178 2 3 <0xD8> 221 3 3 <0xA8> 173 3 4 <0xD8> 221 4 4 <0xA7> 172 4 5 ▁ 261 5 6 <0xD8> 221 6 6 <0xA8> 173 6 7 <0xD8> 221 7 7 <0xA7> 172 7 8 <0xD9> 222 8 8 <0x84> 137 8 9 <0xD8> 221 9 9 <0xB9> 190 9 10 <0xD8> 221 10 10 <0xA7> 172 10 11 <0xD9> 222 11 11 <0x84> 137 11 12 <0xD9> 222 12 12 <0x85> 138 12 13 ▁مرحبا▁بالعالم + leading and trailing 10 ▁lea 499 2 5 ding 501 5 9 ▁a 266 9 11 nd 284 11 13 ▁ 261 13 14 t 269 14 15 ra 409 15 17 i 282 17 18 l 319 18 19 ing 272 19 22 ▁leading▁and▁trailing +\ttab\tstart 5 ▁ 261 1 1 t 269 1 2 a 279 2 3 b 344 3 4 ▁start 453 4 10 ▁tab▁start +newline\n\n\nruns 11 ▁ 261 0 0 n 268 0 1 e 264 1 2 w 419 2 3 l 319 3 4 ine 415 4 7 ▁ 261 7 10 r 288 10 11 u 271 11 12 n 268 12 13 s 263 13 14 ▁newline▁runs +mid spaces collapse 12 ▁ 261 0 0 m 320 0 1 i 282 1 2 d 273 2 3 ▁ 261 3 6 space 389 6 11 s 263 11 12 ▁co 326 12 17 ll 412 17 19 a 279 19 20 p 352 20 21 se 325 21 23 ▁mid▁spaces▁collapse diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-bytefb.model new file mode 100644 index 0000000000000000000000000000000000000000..6548571f49cc64e390f3ea74a0caf35ac2016393 GIT binary patch literal 250431 zcmZU*3s_Xwx$yta3>R;JR(Fk2B-U8tG1ed=@Og_O8f&bvh8k-;h9FUjhzdllAgVY$8eMqYpn4QkFmyM{QUpcjFA5K$MgK2_g!nR z>$~o2@1dW6KksR&>k?*Y!rv3degpcMrX@dinfm$l^O`pCM`u7kpJ|CP$vvUH`Vo71 z^sirB;fd|lPX%~x^K;KVPQa(fBUtY-N$(M&_n55r2-SN`(R+mHJ;L=K5qghVdXL$9 zk4U}89KA=B-ea!LBUo>bV7)zp_4WwX+ap+Sk6^t$g7x+Y*4rakZ;xQTJ%aW22-e#p zSZ|MDy*+~U_6XM7W0Kw;ll1nOq_@W;y*(!B?J-Gjk4buaOw!w9lHMMZ^!Avfx5p&C zJtpbxF-dQaNqT!s(%U0MZ;ueYJwo*M2+`XkL~oA}y*)zo_6X72BSdeH5WPJ@^!5nR z+apA8j}W~*LiF|s(c5FP-X4?n_L!`<$7H=dChP4nS#OWYdV5UP+helc9+UO+LaFZ;#1(dra2bBUEpXP`y1u_4WwW+apwOk5IimLiP3t)!QRfZ;w#DJwo;N z2-VvoRBw+^y*)zp_6XJ6V~XA$Q}p(jqPNEsy*;Mr?J-4fk12Y4Owrq8iryYm^!Avd zx5pH{J*MdGF-32WDSCTM(c2?TZ;vp&J;L<%2-DjmOmB}cy*h|t?3LT`@$+hdmA9<%iJ zn5DPJEWJHu>FqH~Z;x4ed(6_?W47KNv-S3vt+&T)y**~@?J-+#kJ);A%+}juw%#7I z_4b&px5sR~J!b3eF#+hdO2 z9&_~en4`DH9KAi}=BT8?ND7`(R^!A9-+apSEk0`x8qV)EN(%U0S zZ;vRwJ)-pXh|=35N^g%Sy*;Az_K4EkW3JvFbM^L^tGCBoy*=jY?J-wxkGXn#%+=dt zuHGJV_4b&nx5r$)J?84|F;{Poxq5rd{m~x%`kDUvzD0ZW!MIQQ_33B$g|^8w*R3UM zUgLl1*5b4g$A13_H!D7bOj#4B9bRqzq+kDjUN~d4#?LZ8>DRZPF>aOC+_BvqVV!pP zKX$ma6z$cH_dJg`YX7;o(CwM3)y1KabNss1+LA-N-M*`|&*THQmZWX@`-g6Ai)HSX06sZ?PIrR zy!NmEec0_8yGC36pv0}NTcs^XD(y)(=?$%-pxj-$n2m8-zn@pS10|tCF<`pX1>%V(L2W=QnV9ipJvvEh@_1^E4$%D@f>dtxeJz3UE=-q)l4Vdmc45 zLHq6(4!0%NXw|<#^VIouE%!Yfs#W8Xwa?Aly6f^O58@xbfvMU0PEMr zY|(yQ(9~0aP4U`i2XLuWQnqNHmiIov)&s(u-D$2}rA=u?_0%YCo!0eAi@Q8)wKJ6- zEm3Rx8qHI(cumMZFID^di%z$i9H;eNgz8B>Hc9iJ;PP|_ zYCh3|jy>qN0db%7WH4X*@C_WE7@K0Wj@`Yk^>NzVk9%E7>$C%BaSiO(m)1(&lK6Vk zTJ6m*@$t0Z`uMomIBm1?_-VfY6(nwTV%)lvn3Q;;y8U|^E@_o^NQI+AzM=i&KANYp zvD&jEkM)#gZH%@m8kcA2rmluWMQy z8g)+HsBQk)r#(q;NY(a@$AuwR({~pyL4RDfn6(Mo_b=ln5<_VFGLK3_{p?j#68*h) za;-;OuYIz~^EhUsR{eXAXM)z*j^?o^DeV8;N$CC$sGc0(h@mN?e|DlL z${R`A5J#^oK1FkWaME2G`nUbIM_s>O3tah`yS{GY&i~aZcaWrw+B3zdp3L9asJ-%c z&x1GPw5s3!+0%0<>#DENJ(gRgjqY>0$FWEY%Edt?M`@!TYr(zRWRHen&Ng~fPZE(H zho@g8#%xSrn7CsP>E}l`SeLXhF=lQ2@Bjb4>o+F79`kzqS}N!cOH*{2Gdw!|X-&+= zm{>L5Myq0Y@|o>%P`ck~Pb+Uqa?b2gf6}kNyT`3diCaf69HSn43KW~PaeZpC_VdT- ztlk)ts7+E1kNMZf#BbCrdAJ|XLU9{c$7wUR^#*t&er=rA^atEnHF~SpgI6Q5h5}$~LpH@!?_ZvXROi4Z*2%!&xhKLKF^TbOwQsG~R5W|(?{rz59w#MR8=nxTy_&}}Pe#cpF&k5~iTQZApC+%-mOOUEy~&{8 z@lJ2h4XH`08~*5_0&LM1Y$rs|1iE&umiLF=7%3ZLnA1-vkH9H9+ z&G!Dz-iT~B$or@JdOeckVklW)86KW~zbZ*vV#VbdZ)$6Fy%wh@n~gE+614M2esrg< zTc>St^af9iS+6~Fy7%$gBxa32Uh8$Is3pDhN4MJjXqIlCKW>Zl>(|nphlhXJQ<|u? z+Kgv#(K0FVvD)cyblQ}`ToHkaWs~DquhWXBd1AzENToa9oau=XtJ0d|(bXjUrx!d? z)~ivuIo1<}K9iWF)&8#6wP}rZJh|63f1MV&7njEzu`zFGAMeLW#)*_wvmEg_NM*ft z>fbyLS}$2!dZ{;7N>Y+`%&()TPV17i&Zl}^iE-MK<8e{LHS7(K{~vTxC&9JaH`hFl z^{fsRPRbI29{a~?mG>U|t6n@Wq0>_g!ZaL3_mqB(HuSSzN0R20 zzxF(#AaCHH-&>AL^-nrgLbgYbSr?=I ztnsnV(A{(uoh(ulwQDKYdm=AcqrLbhE_Y4hw8ek*=&>~Lo_D_P3AKj8{{fe$xG7ra zr#MJ!ZL${Sd!xtyP3^;h9!H$oQvCG-4$@tZ{*V9m=#*Xi^R&OaBmX`|8|Ol0)_zla z>F!;(_A9OW?DuXhL>rR(z^(ndS6i$-`PlCn=4Q{+alP7ey;^9m7SXFs?bTlB)s}iR zYH~dHNA;s0)v_Pe@*maaUiIw3UiF;iM^$aGC>@pm>w&|)dY5JIKk1KS5~eG|)$9LN zs(#d4AD580*NR(v{r`4AEp(oIukYUO(N*0JYQIq4sAp?`RO>vdDu5+J zoBB9nm!%E_(f|4kG1*Q_m8zGEvDV` zbe4ANv9?5;^cRoc25sMC?PbmD%-~}`S=?`MQfkUX&pvj-8>ws8{vu^_%1?$W&y@Ie zTPAQcQoGs-uWw0-d*k&d3{QRG^<2N`UpKxz^M4JGKOp&EgXhDMJfnPIG(5Tcynnww z>k?B|u6rXP_9waL{iVeylBK>$5wii>elSX-w}@pl`d8@FMN*aijmJM*>C;5oTSWfY zSImdtFk}W2c^L5^Y^0bc!aSG>=3xAxlyLdT>93e%+6yL`-%rdl)|kYQO%V(RG3^jB z=8Jgm7V+9C;(I{EXRk=#a*;lVMEYAr`W1`#9TgeSDiQ$x=z~s(3_K<>xLxEYr$n?1 zB16uJ4DAs4=_Qe2*ZAV5Q)Kupktc761l|=Haa$zlp~%SlA~E58WhLc|ecoSQ|F)0B zJrf|S%>J_aivW4!+W?7QKS0(zH$W1;9U#B^Vt^#R;3sQG_({^hpUAqkezJ&qtRFB~ zlFJ6m#%~5o>N5i+<;8)r>FgkRbI3s1a`vaP`MZI#lsta_-A^S6xf;^b4D!JMG0!p^ z<=g&Zj(kF7_9r4yFNn-}O=RA?J~B7UAoHK`kr&n)WI?#E{PJ60S-9R;UYz4Ai@x=e zU$uEj1$KSuS#MeLQ9t?hKyQhbe)955{bcFcKJuG^{bkt;edLwKK9V&^a{rY7{f%CYmHC+9Z|;+9l2?<0#`h1~19KV3NFBCMoDE(nQ>; z_-{v_h0G5&$xigPFGLoR<|b$uF6L-#V~!&&Khm5ok{M`{wt+@zr(PN32%ln-B8!i> z2y3pT3~&N*J}m|;8|{`fdb<8qt07I(kYV# zY2Iy+ws3>A4>8F0V+P5_-$hx9G=m)9HB|nze~6sO8Y-W@G*nK0GgSV~{1eF~znz3T zO+Whc)t?ggr!o(lpJ{nY{<43VoNIep&VDgQ&VT*1)Gjv2LY|d&_(}uu{-fn-x%Ams z`Qr1R$>s0I%2-vlTfVZ5^xD4nlA*YR2$y@xN9>eiC-GX4LHKVW%tT~5&;O{}`o50@ zBFz_k#B$8XZI5;@Z^>9p9aDY9Y{mbmkK}~=r2tq)hoDC-KiHr)@5XSPi~%9`{cAX2xEF zoFb4NYpDxi(G-x4{DevvwC zqg;830g_MLCy4WJD&828m8u=ER~#}Rhjp6I z4v>;|+VlkL8et180kSg($c!Q_(y)NJ*dSrFQ_cpFc%EfZ zwr!foJJcf|TjvqBfbu8ttPQr4M=5i5$$W!sqU<%4sc0E%LpgmCEDGvk2I~fcq!Ff4 zrMm+=lFv*i!rn`fi(n;K5(y8r-Uc~Lo)hs)ryi-qDbD@Zf@VS?0y9wk?9WC^u4D#Aa zo@aP=5-zGV>63ZbBd^m_MzzOi%OahZ3{n(jlB@{UaKc?zVZWr^UNFjP%KwdOGtx>& zZ@~ttj@^XG#jofbZH<2twB>n;C6%<{9resX`og;ZvQh34{$2dC7$@%|AK~_;Pq&@* zm$qlAk9bS_tKQO0UjGAaANaG*5x$59m=ob-$I&K5H^l}W1tK%==44Fe39{ldlp-@5=QM!W}wf3CHQZJX7)v| zkY`4q+M}?35A$U%WIm%m<&I-M-03TKDc?Ks^rcHal84N{>npz|P62E8_Djrzr1yX@ zqe<`oB6l7lomr#I<$%Uj#xiS*53HyKy z1N#%~v=6^rq+Q8D><_F4sYGvkiSn?=v(RqlQ`i!z=6~};+U|lu3JJ5F^&&r1u8L{_kW=5HAk~D(QzkY3EY<{5I0wh2Jl+ zXVDmgyo9V-M!Vpz#!)lUsu|=E{^Ln62fsfMemMGQD5Y)8e&k0V$PK36^J%kTp0zoI z=T8yuL&94ZWB1SL$KM;IY?jDWp7o4p%Po^|gfN%DcFOjMep7{RIl=g6tRF$oK#wF& z)+v*_kK~Zo65Pth?~%s>^xPPc<;YkFpxkP%Zm;s}57at6m~?&$!(cmhb0R;5+6y9o zMkW*QIPs3FKK`6RenYyi!fBP3mqE@FCV?=e#Ufv#7ww=s?>@8h;Dz-}XeHv_EAOb#D3& zzxGDf{wg2YN1XpwVaAD6q8BM{5&n6?vlp`CZU#$V`aja`f7U2v%mF`FuXe@W|SaGJ0?#~H+jdRy9q3m9f%4)&p=1U@Di1Q4nwI?%*K8sXq zP|ga2+{9K{$S!0%XUev3nR5*zB!7!R9^h9H%^DM9kUQv>j*()~f~1f-Tk0Oik z4<=tL>1S=Atw#k(?evjSOP-}Xzf66G6aN&n)zRKHBjk1bYpKr~WHJ7ODPn&Z4a32F zl=?%)DD23dEepGh$3G{Zu|%K9V+`ltFvxk*-u4i?U1Of7JSM^vbPxxd?_>_K5Z0_X zgbw}5ZxARO^rw#byA86>!n1Y~0(IBY{w?F;8e@XzmRpS5Vl^)^K40Y=9w_oO`};<1Sz)&pUi%=Uu{! zm$ZMwybwoysN&1SNg&*I()*Q){{*&-U`*nd3~jHH))>x)w5M8s)tO!78yb2(d|$Nw}8Rb4Dk^(D?79p3J7 zHA30>s!=ZUJP(^#7E%`YEpfZ>dkwiW!dIRrT;2&^smA>ddzPC#+ei6xulY&>Wwx+> zn<0{L8TYyW;66a@fz8|@sJ@WX=_6Uh&7v=668;_5{B3*LKWt&WJHTBJwkuf1{vpv< z?vQqynR(`Eqtvk9nM;~mi8Gw?CnI-|j(G?6qK(_=TkT(a$phjP;$Jl1SN0-Jge_|I zmHx=#6yxs{uRro|F=h86e{aIzE{pM%*Zjpy_cVXTv^J9ciZ!0V6R_`e#`Fr>n7z0f z|Bb|}f?V`G#&*8LAn&vqq+pyTEu=S5wf|z&E>mQUQ<00{fiG=bzmwq!z zg`3Q{c+N}i^Za1ECgJ#PXMJhs{-uraUpt>O8^uuf5$Yc3qrIBE$u-?v{Ic#+?`Mer zte5;7em3UsvNVGn@e^}Jyg{O=Lmm4C$2j^&UxN%G?Z5l5KEV-hgV?7Vq=htEAF|F~ zV!bDgMg#X>&;--SBN7<%Vn1M#Bl#w&CyY7|ti?YGH1hb0^sYlV&+@YwYhU?_>kwy4 z@-tAD?HINFs{!=g0g}(YA}^YA4}Ibt$ab)=huq!V0a_T}gnu4hgb$u&uAn_iIG?p| zr|*~fNOnATxA+AEs=4i3llUS-;d}C{-7T_(xOU_bXhC=JU3LC{9{DN~ zqqTDv+t%VM*(xIgXHvfai6y-z;$(Y^B%m(=XEuFf9Cra@0%Rrbj8OrSJx-*Ke6pD< zR^w;q9DF1@Kx$7Jq=vhL2F}Ly^c@G(fhEpBr=@Sw7aP^Nn#nwieqin--;MxDB8{cc z_8I+zj+@P#n60s9P{$Bx`+AtP*A0`yE9i67|Ko*5&dUMvQ8fEW;%CsN8?ey~xXzeM zLUtp6kGzSj<9VSUYc2X6$ht5)&D6m{ecGO3JW~&IJMR1b?sm$g{miMfK`rt&@&eu@>-3aq}=4M z?#(jjBg=8Ov-ZX!6CfG#*yA504fpx1h5aw*g8v}x9QFZwcy4A4TOML(l`iG9%ooml zjNdP@0rDVW_CXP}v;OX4UdbiBcFNf{09zd;E_q}!e-sv@TS?;rX&gY7vt|(X68WjN zGM@qm{ub`g@z0F+;vSB6t~1G2l_u-%1nOF7@M8YpuIW3Yxc8NmYZq(kY07X0&V&2> z&c0jiQ`H?>5%HC+&Ft^gIjwySeP|bH4D8E&>QANZ6@Sk8^ot>W%=7)|AKYVcFlxIx z6mW<4m-g@axck?al-InNH5D$?R_*W7e`u>N^tLZZ=NtCq%3ts&&vxD`wR5M?{xa#W zA@8-cAJ11Y)^_b>Zuw-Ym>KVuOY{ZG+P;c#n+f+8epdQ2WXI8l2hbPe2fI3%m)Ps8 zxuNg`=Xl~@A>jhi9W!$y$ z{0?9gcRh2}c&BZ$Zg5A1o^y@$NMNPW2Juzp zr@kux{^*YgGaTAJ!yaE6C4@SkRQX0wE;xgJ5l+K-BBL72i7 zoV%1C;o8^J&r`f*mtmT0<5@DavjjHex(yFDBf4wQr%nGPm^HFPP(Jhy@}^Zzi;dPm{Knr-|jky3NC#cYlUt-<~Op&?6!H#7y>OJRb<(sda#MrZ2R&u+DSW-*#24_l%b{NZ!S& zvwHhx{JHaQ=kCAlYvgC_e;3Y_%!fucD4gYP7-a!zF9EanbjeQRTtIopVn6lHuSVI6 zHQme_q3oB@j>JY8YK@_vnU?`;y=wn-(r}lb^F;>rQg?#x=i|^fU|+_vnh#X`T*9h3 zUG=R@^lgwpzV3H4UrnbT%r{>XX8`H5MydOa4EjpuwV7ffY=0OG@LQC+ZVJY_dcr$Qy4Ac?x&QG zY0ta~KNY5wG~Dz4A)fP0_2Vqs&xXHh#|C5`b$efEo^lo<^SKAP zsL~@1wcfewk;pqO{F;fklRV$0uN8JomG=x@vYT@6+C5dS^Zb44rRI`C>Ux~8-T0~T zdm4S$UH05a_q|mI<+;Y3!1FE@=7xCkAwLz~MYx;F9~+%SzoT^g&!E>H^OhRc(+2b- zNXI2_scUDgrtE4CIm%kS54`|pQzw=j^B>^>^Tl@LZ=o&EDDNTLug;*a)8`pC_ZWK` zexCuas?28U+5S1tznUS*xZ7IL8)vX@V?V$iA%lE!;zdpnKASaP^=}t?+t=82+Dyr# zero;MLfky;@h6_GLRbAaPo;l>J56LxCu5lU|AptO|L#&@7^8W#S>8DM74y&M>KRg% z^Fy9zPvA`Q5#?D+c`i^M>Yrg?kHMa$Z53r{Gs@}5>9q}@Jbf6egGg&X&ufTR$Ncjt z@)M-lQ?HLmSG}8P`+#}kCH$YlU*$1C%^R>8yr|bQ5w_)&2fKZ0kaOhIjQg01KZN*w z7^9z2e%kA#3a9*^F^UbT?tpWsmpPDgV>|VuPqa;A&lO5q?2U8S->p~Sm=oSamd^2( zFyyvdgax&ZI zbFu(w=3XG@A>Rk6aI8gnjG3M2^Fir*l>ZB)&lve1!t=V%yyp{dd7m|9_Y)k9Y5&4k zI1`g+9%Xv>`)QJceg0VHEm=__b5;D$jq+g~VONNJP~|P9-*`!$YKt#Om-Vx##Y>7* zxYzjR2>Bv7!P>`5K1Ei3=f(RP%I3{|G46)1IkS%N62~+6bB?nkw{0Q+G~Q*Vcu5X@ z&3TyfMk;=RB4>%y^djxfS)uU<`bZgRVaFr*t9MhmY7ZE0l;?4`c`+AIrUIV-8h61k z%E>&TUhnSU?4C!OAMhS~@3Ztb{Dxzbwik`qgY+NUkE0#;Tb}ZkD_!}MUKM)oUA{~B z+$80!r{{5BqRyWUWUm0{U{@&i4CY&bHAeXh?n|V-i+X>KzIzn&4C7($Al{h_6?4QB zj8&vcV*V zH%2-98TZz&y3g2Bpu%Sv)%}MQq96X=C|AGZ-lokYyU%i`h@btcNlJ)QN*Uf`pZQz- zoX7ZP2e}8i2AKqJf*L<+@y3+9n5le|wFE*T&P5=HjM@J$`hYw1t+myEKT@B?t?FdtsUE_V98T3?^Q zujly)^nox0RGvB8$)7Q#^4iU~f1fe;6wm6u#;&BP%KMB8`?*o-zN1X4EFT%=>BoM1 ziQm98?&suL{GNk%=!XrH8E@EgGVXA&^L*N4xfr=0n;n2i^k0H{hyDsO7JdgC;NX0b z&B%0k6>?w)oPiT?75)QuK_+`4%YB1;NXZV` z`X+j((AVh$UAqQ~I)CiRXAd!id1NzltawQ*N4eFI07;Mx2e4K5_BFEq5bynQZ-tNd zvgbo)LN08B(tPG;WFeH#H^?4jRXKMU$b(P>_Av%2MOK0h8aVIOM{s`>OkY%PIC`Ib zz##6AhH)Q*+XctrB$#Vhd-7Nx(KA`6vR-DceAP?N;LhTGxVeM%bOQSm<{>A2>mvS_ zA(yd~v)@bXti4xocY%}oI!Aa*J$nMjCBEmuubDNt>9$EeeC50$>nit#>`|ONYbD(l zaLN->%-AjQGs#WDXN`D5x{-IFh;yi=By@$M(e5(M(1}Uj4l~y-05}R z@a|XtFkGEukoWq`Fg_x@X&Pr==nu}S5fXqL3`1c!1i@$+3znKhjL7}mh+j*_7n>ye zsShRVU8m&sIl~zFo8+$zGvr-;*0Ak~w+-)nG1#!uh^lEJ9h`vuizWSXx>?1Z~HMHi6yN_s1Nj!)yM>J;NKMQBh8d6 z33oC$%SjhnpcRTpdlUX!AssTIWEnQVKNr2yK;0(r4g$RZY`1u~0amDjGV&;&V2}#N zQRQ@lR3WXntC2Rwbs=%~z&r1=pdoin&d> zU-OY}5LQT4>@Z_NG@}F0Qz7U3d5o29z^xmAlzeNJWPZT zC<|nro@J5>sDvu8&S$)DV7x=tQ-KmjxT!D`B4HjZfJM-FgSqW4-y}a|oxP3ycVd5V zGIzS5rjz%Ba0Kj72aa1J^_N&{plg7ybUp>oF^Bi@l_jLP6qdtEh=tXV07=mOfv+SZ zHz_@jKAmrrt?23CG(Rbs$a?%7kc-|NN875ovUq)YcF7}aDWD8@nIgv53#Yc{A@R3HO`}}NWJ}74GA^m-D5Q?A_Y=o<3 z3|Wa+1(g=Y!(RIQP9G^ljwVbYR6@~-K(QfTp^Bc&3mB1Vc8s?lv@ zMv5K1fpE>>g5yyCaHKfkr1HNsQkwW);57OfaK#6SGbTt{kuAqYO4;p^QVtbR300w_ z&pZ8Uq;0}TIZvF6a2c*Z7hH#%&<%~(g5>C}AZbE2-zUu*q`i|oz_lYtYNGi*V8uu| zg0w#jk~+dU2w#tE=uqDs@NEL=+=B=32ufOmq!>&bCoA_dcam049&7rG%n8)HdK6_; zwyMB#{L7E(VLgC|5$7i7y5Bu|M;%1M1NmV)!PZ5k3!+u5cJITrQIKWw-*@;U;v$9k>V1X?^8^ z^2-`7ilReCd4xOXYcI*=aAulG`=!x-;EP_V>P8*=qX&TPA$<$1P&JD(u}*doe=yI6 z!f>e1XFl5DBSGlR%o9zxN28Ag*Ini#+^x(bEy(fsO@t5#gQ+kRBB47gQ05^!*%vNA zcKzVRM={tf(6fJ8gkRZh<|3%@V{Y%_#0@F<>)IR7FI*! zAm+khl##i!8CveLFW?TtiFDmy|FMv{G@3be1#>I14!0wiITsqBh;JDZNGA!BVH1?( z`%3W+U)hRY$@hyDxYN-y!A82(xUClU56E2n?49f{kOfeG%|{B6d%$s2@d2!igiC&kH!W1YJUG8%W+2-aIa`tJt%FO0>n=&nh|BPT)@=jkwH3FEE!nn|Xj zS27MOQaLYwhW(!P6C1Xxrk>Vl>I$7N@mm+dbq(S7DW43GNc4HI02aX#SPIKwCB(vN z=-xbBoZkmX0(uf8gPk_tgmmnsebKinf5vt?vWdRXypaA8P5)Ry|3JFt)BdykC4}dh zkPF+O1i#{Fe9LK8GY3$%h0Tz3PchVKq)qx|K_LVvL%>sI)SgZkA&1C%W$|7GM) zxXJ_MPyMXu)ks^bzg#4b%Wws{;5yudZny)Dl>aE@Z?gJxPQkv|wiW3FwLh;pLH_OJ zkF-;_I;6uv{-oW2w0rx@J<@pqkH9pGG2}-W@I`O#OWFp1>5m=&t_}X;#NCR2%OHQr z!2YTY-S0u;`1Xyq8cMj~P>fy?MZFgC-Ng&)PC}#_w=I-j{FXuyXW-?? zV%F@H$XF=Z!uuy=0#s7Z3a~Duj$osV)lhbU`gibb0_VRmtclZE8WdC)a`S%9%ZwKS^0`qSx zYyY>5Ez;>4!uK%9o^$0s^b9L^`@*{Zjkg@cT?D023Eh0mW<%Pc0kU6BlV+p~x>oUf zLS$#1K~5sOKk}E;$m7hJS>H2fqo0S1U}w$AL|%p~&;|AQIaqtIqc#Sutq`^WgL&6oqL9f$PlQX?<--* zso>yyj3#J?nYbgt8OpvOg8PFg_6gHD|C2^JYh(qolKo880n*+<9<{8$FpoG3U=b_< z_xxYWd{j|IG8yX2Sp$$; z!GV8MBK20_6*&d>Z9TGs$N(4&Lm_KdnmDW8 zlHuqV!eA#&Qq7J&)<@zxFcaX0g@mYx}SPWHX-f#X|ffW4w;Y(^*hoy zKcvYv^ya9yr3nhq3&Dl%gx2V{*^9i*Z&lJ{5B~e$AQVB#!Zay{QuNB~x1|Cq(QOL! zYOwBpTdF|KKlb3ad*`3}+v$?UdkH&X9izy*Jzbj5rF-Vl&(*zLI`eWm`-F7%3F%UH zEnUj*rc1@cbg4vE-AWg0D0xhOTWrV%(rE@49EX!|8qUCZXuR~c9PK0@Xoi++)a?d! z-21k;pl0FQ{4V=#IRbX5yZg2{?!PVdglT{x%5#x)F2fb*f)dBuQXH2q*U>9?Q~&(8 zUe*9v7F zY1BWB`loUKk|tGw^nU~WKZ5=b0mK;$Lt!|$@9~iCbKD@#zk?Y6!x;ag82`|6fpxiq zc?{`d9M#NM-#9b=moc`XZXx48nDGw{;N*U(XhWL&9fVr{OR|kJmh{I%@lM8NKI0N{ zc(0o~fHixEXIxg{XN8G`%leG(j@rCA-w*y&!%OqjwQ(-1FC2~H9W8aX*{DgFZ z`X)G%XY;_tI$Jb`eFl0lvSb2x8|X`5DO6Y~M=|q5EBPX;LCufLdA1T_VKpQ`SDQ(a zkm?>a8QIObX%liQq(dg;!Zzq!pDwX#{CP1(aPLumoHzC%yIwKyLYMgrGR825zj|8^ z;$H;aSKpRWq}{;21)1|TXVGW9#D-qa{O<5(zCl-aqD`ILbt)ZPtXs}=+<%Z}3;v_= zD^&h)91a=07ODC=-A>#!f|?l9I}aQ0;#2K8J2ljM6E&cJynNoDpZ&&f9Rl{g41OC^bNA(zy^75Seg{J{$9Fy*8Of@c^}*C zS1!m}Kz|(h2V0Pwb}uFi-kMQNNe?jrJb^>Rxm$;w>JGa6S zEA8r_osqT0wCe%d^)BsiOR6Z@ndiFkvtiWoVEc{eV2QU zhrH+5zOZ-bK8^_q>cvO9$}Aeh%-o;P8h~^zWB$M8C*=?Qq~b2;^hC~u zshkVbIR7GTxRZ%%hX$1fxZpUPgwxP?m-FvK&Zx-dS)9M2l@ErT8#sSMjmG&mkn=Cn z-p={=0%zL}&cDb8+(l`Aa)xxy!$r6ZC8SpjSI{djafUs|JwTaJx^UZuQ8v!KR%F#N z&bx6NTZ#~>Z&$-Gz#*6dm3hoCcP`*&g7eV>ZUHmSiESmHG3i8;^ zIoV2@4$^KV52OuuH)-5~d+-1rfoU%180ZhZ_rHwiW@w4W#^7Wx=(@oE?*jENrv9X5 zKSce}9q9E3s6X~E3iOu%(isdU(jJN|nNIy-IC^CaWuv}9=%c|FPucLZrc%}|{xTN7 z@h}lWAPlC$Oo#;cc}2rM?1|^$UI2?Ai|-TE{LnQdK$ha}yz0j|;7-1VzB+_^R{T~% zEUbnENP_N90wfu^3AREyWI`_3S!1^$)!lo2D!&gzFN8gy?%f-wh*Q$ioT1y4kgT2H<8^?9Bq_4$b0Yr9)W2d zeGB?ScNTX&RbJA0mOg^JYY4w5M-GMI5CnGm*=Xcg7!MPn{vqo>grTc5NfTsU^^&Q$ zU59A*R{AKkK$$n|d!XlxSvkRTrnXLJeGg@QALk>-DN_dXz2%y(m?4BTWf+^ya|=SOg`l ztpBhCy^`k@uoQhc*g7fOL-rt01uOBhQ@&W_YN)@(eLgY?9F(_d2kYBT-r3^b1kP0U z9ck=CEUa(HGRjd-xhg1EB~+nXp?VATPxXQA_OWCQM^TJBIdpII2A>Kof7*t?{h`XAz*0qIwg ze#I#tIf(9D!}*^wRtHl?Rn`+eQi{L*Hs2bp@!Sv8uV6ojpA8(*wEcYMC9vae0Oxe( zrBSqf1brZsXZX1wi#gJ19wEokJGtjRiR=>U(B~I&8vP8Y_2xYCB3yxQ}+y2gh;tKsMcAY$MekK+P>`j;US983gR3ID<@J z?H|J)AcD1j5PJahGRA8;<*b0pYvlcqGHBfY1!8MI{+}0l429tk1fyXrjE9NP$UggM zU+hfyX5zI3V^?sW|L-#YvnKSO|7n}LD8}YO&i}{;+9KzpK)&zb9V}^e^UY)!aw^PJ z?oatG?b;F2$?p>)ad*{lAB9xsg9XYxhdwooa|-$rSPI$SJj>cj9_$B}<8D4d*&Z5X zC3-BlPEodFlo8oNxS~seG8q5WkN`D9a+gc_bQ|ncQw+c z;YS!7*r5TM!3D?RBsBW*{(l(n|3~ruA6huqsI!feiyGH4?q4_XKLX$g*rRy=Ka2PO z3wi%Po%jFzKLI=67@Q`ZGjJX*LOp&CxQyQH;QIx*g5Cu#-t9ZHM@s9?kv&4{)s+k zN7k*NZy@V&Hy};(8CTFB0$?z7^9^SAHzQ>zdgr%+(sh;f>$#EAnPrgS_yxge7z=jB z&@S%8#-mS!5K!Yv^}jImX2w?&eRL}NOmN|LUgC~`@z#Qj#BUxffTA1pWn}Sv&fn-u z(3iq;sJOuRWIm|UJadHm{tve~2D^jme=7;M8nVQTy$ySE^qfz;B=;rG(`fWw<_?~v!A z9{nEj0oaSZFJ|D_Fl^IC|w` z)~_wx<)e=V_xd%T^=lUE7qX1!<;w^|`jt=xR;UIWj3v%^m-+}JttU9QwsVd}+B-P6A|1Hvkquk$yWuNSNoOWxeL}l^ zLECf^2LDL>-1RhY|DMOV<37hly`8vQk;*Rf2vcPBl?BK}P*Th{WXPpZjDJNG^;*as zKkk)Kja$9fuZpIg$TI3)9!LEX35V3zUw!`*OT5*P07;Mxn_w$6?xg>*7B;c3YlfB$ z%zjuwV0*bkfO$T-XNno!->nTME#d8OKdfh`tA0 zsf=F(RC$vHfl+9xNQ;rJCrM{J^noqgX*n3b<+flXu@ROUw>xMg! zwRV8qLq32<(8zc`%D8R<^{rnE@iP$&Oiz_ftz1^poa2E$Mo4ngn@_3!348QotqHgK!8SIvP%*?hB)EY3H|c>E_q ziGy>=^L%&k=>VyueyWaEr~(_PwkX@d`FH2z^KUt4O3tTN&ZpH#8)xs)q!R*RFcoG( zB+P>a(3r~m_cY$WBb)hu0xc&vU-Kg;C(`wh^FL|Ua+W><_H53-oO>Od)9axD?3{lW zk0%2e!dh-VE783m>7JW6ic9Zrp-hVIV{Ws^}1pJC{Cn1xeg!AzxX9kf4VOWm{AEMxq^ky~o4G5#Jh{*XQ2-|Qg`_0CRx^Sck-KHpys zB6Gr-Z}>f;E10w>N9!okMmCeS`qsxz-zy?q(J+50MOK0h>`-!_?|*O8zmz{?z5<%j zU0{o$Zd<712I`7Dj$ike0df*~8qUCZxCocw3UtABxC!0L{k6Z`L3XMi5OQwrJwHFs zejon_@Ccl052)7PIecgHB7GVHU@#1Y;n4lGQG$@8VJwV?i4X!|pw{21$e9ocUHmR- z9JuuQ9ZIVUEC9o8hLwz9QA6B9__htM?@IQ;tS3~djfA`rd4PyLT82^l!iUW-Q zTE@l&#z+Ta18Hk#ero4_pfB?i_u&b|Pl8S0h~qeW_an=Oc6%^nU+$pY|Wb_>X7(0kuX~LKV6dst>SE?e>vm z;v9#Qa2n3QdAJCdp^}zm8fa*l*wU~M$t#Pcsq0~Qua2DzhRbYi`u#xT~;+PgPCqsV- zfWa^nhC?Iub-({7um7k0Y2V)W|EjLwe*fP}{YkrlvfBHyHzJ+UFc!u`y@C4UKM}oI zqy7+r9tJMj$hm>~dsBayieJ$}z9B=JC~HX+XGQdRP>g>Cbya(?1-KW1`(9@{^_)dL z!NM6?{l+INkKZtIX3nXZCb|3`$jR@3mJ)tBtb|xt4GEA0$*>8!zU7Vt+4&r2RhcH~ z=$ViU+n@llCr_6`A-`sD z!Aa%UgcX)OW9>60o{ffQ@{h|AN&L3;0N#``ak-@FLKOBr43d11? ziZuRzGjc49hlx-UVUprt-hZN3GKW=wbJYx)id)TT)u6s{sDiS<8B#uGhEz>T>D@RbYhz?=fPho?CgA z4)x<^NG38DR5_c{s8`es*@n9SoRrJeO5W6~6?rf%T>cmrAqQ@U%l^faW#6(;`EYTl z?7jV*eBc)>d*XuS{bSF`?qw5X*TM->*!irycWIpLTsBVLz5R^T6%P|f#}iUW+F5-# zukX~P)}XoH<=I<@dG;Sx#%3|U=_gDPd%TYWr^_LJSo+b1X>#}k-|Eitk`hB7Dedel zOK7h{clkZz4ekv(c%Q~^zN{w(N%bY(8UBBy-G7vg*`4_NEy`%4t1BWRg05I%v=I?s z@ine4m0n^QODu846<1v8E3R)}?TQg&e6=oGsfrOJqD4geikz-eRi{p!DyL5UI#uVy z5=$)8%ecn!wZxKI6<@K$t+B)sOSJoXjwYSS%$+~(AMf?~?0ufIpJzYMv-kdd_OEB! zM?=lZ=R$3t=R@6P<2?fk!{1N(T-bGVayTUIBlw5*$zji+{~W$M@;`;$^Zt`@=f4Qs zYsY(k)Dhn(2o*;shRPwI3+2Q9GVIvBEYbFsgywZ6p;fqrjE#CZoR!WwT)-t%Pk1?0;R-$e%FEium%}xB50dRK>$iMa zKjzEoqL;%Bakr4#qK+V=b&q$+`-s*-J|rI_b4neAu6A`$TK}YH8jU0Nov*xOAbP*9 zE{@tleM!-J%iizKc+&p=enu$rd(Vg&VJI2({|%4ApJw;p>l=#cmF`o;>jhyny?TlA zzo2gzOONw$>M))@5y>%pF!e)1{ZLFQT%gsjNzY8Nn)}dlv_}&z^5ImoX7tZq z`hG<@!9WyYFoq&~+j}(6yKvq+aoxM3k0*1&dpCt$cF}w2ezX>{yS!&<`teEbPYJtp zf%g_QY~xzgZDU`c9?`s_XY-$5FAT$@^L(Q)6qBPd7UMAy&;0*o?(rheWS(c^Ip^1T zkc~&#{|WYgmHX4q|F7l$ALjq>XP@&0x~-|~{%3sv<^lGwjV4QTDyE|p<@QyaNzTSx z%*R6X+Pkp#SLV;m`AAqSycEl^3f1DOu$mqpCvRiCw}0(9*oQwkOx`BQBkJ|T8-_jY{X`?UF82?<^PkJoBaO={C{NWIW%77 z|G#37!&!X&WksQxOwZ&0ldaNCjp3V1XFDpf3oYW(Q>;IrXLd+?82^WU0J)vg-sT)+ z=iZ`lNL=hyK64~dO+QA)QMFFKmdV$NqL36$q2Uz$J^sJzO^`L``Tv(4H%c=S; zXi$cu^}ntDlR*wAk?#-Sr$lRjPYbs>rroiB=zr7yu8d@nL!)xmbnk=yH+4w!RpmeZ zhyJ&-(m01yyLaZN-VJ5?g776oYk;4w|IN6zj#1vNVcs#XU==rwqU+^EgzhFLs@OVtbWIXFX8DwlrcshFDEefUNY}5?1|=WnK`&s+a{QGerm_CS=za;!k=g!KYszJKnH@^?P z5PE-Q4@EMDW9a#)_m}LpA7PT*I^l!+!`(Ie}*3AE$w_cdggaiPH>7h5y||0&{np>9ukX|2 zUHZ_|{li20V~zA@=vzGwpHsZP-r#@cpY4 z-i1BL@{JFWhj0WjRQ%GqfAfZVt)o9C97hre?Niv^B~f1z`=by0 zLtPQA0nVulQrgG6&T}6R@fa6KwSD$(Md%d+7U+ z^IP_^bDVcB@_@LQbPkb6P`%A~DtQc5d*!Q3zIH%3iH5`S<(PUhvETUnIQd82BFC%| zzd`&K`6rXc?^FKMiVSi%iPJcXb7;FM|My(uBl#ygZ@V_<%_4`!k@A0C{>7)rX2+$; z7U5Phb=4jR(z%2yxQ3_uCzO9poia+=@|k_f*sb4k!r5ugG0C~er|(~EzqttBzpC_$ zp~rEzP)*zZ>b!D5@B3BtFUqBnwf{%Aws4?uPcVnz_r;-z-u?26q4yW&5PYLJ3>F@W z;V4FJKQ^dx6esJDC}fwC$tmnql)v6DjCR~ujK@Sw##Bs4`%Uj)x3c{}87I4hJIS27 zAWJrlGyi7{+q;AfK1#V}|2DIITQBob23dC~jhUE@*bepuSw&W3K7BEkqE5Q;>*}9J z>L2MgAbF9$K1W-#h&_myIwAYR0{t=U0xb8x3j6L=BEP;+-CaM39neqvQm9TQS2$)B zR-^Y9z70jL$Hpl99p58RPj>%C84>Oo@nR?=BmaMU6#l&WOWt;T<9~!o;a%8+7WH}> z`{*&1BvNQaO&{~RQMbW7Z@zcEwm6|pZm4A+ zpix`j#0NY6K>L1A`+iXSj+UL;_ig+zvK<|3*k>!*99!5PWL7vwMq?{m5BklyqB*KL z@+3~RJwRt=A*@))c&L!t#Ar>RQ|AzRb z^vne163gi;kQ=F7BASESscnwGr)|e7zpX|zH@*Rh6Uy2d{Zm!u?ucKHjo6Ip!}_iV9!?&(Q9iKZIC$FEt|6joWM-nmDGgw-aF%{EMikX;= zw%g|a-}_+x|88UQ`;E;XHgB2C2{%sS|4%di-^chrnji4((V}0V^`i0jP0o+H-9_4I ze)(a3Isd+ao@_4)bDd*87Gg1$VmVe|725Oxv>((*a8&;TIv400(9e*aqyGVoeEcT9 z{&9U2DY98O-K75kt%%m&ua?eQtj9*QxX$#D7s6(GW{|uM&__Wp|1qCxgS-iMIi{0* zOIgdWzgL!`@9$L4^1F6>yW^|ezq)zu>muv#jZH@LhZ=;F?nynFaBQXHc3}_p;Q$U{ zvip97tZ|N7)D4pk>gkDBJs(8<6VLh=yi=)r{0q-N?YTakU-EDEPu)@O5zP;bIma=? zkwgm7_=ecCs@TD-%d+vm1mHY?EjW z;8t~8v=7f)^Q>v|d-pd#W~`X4@!Y@=t-mW0*K6Ox-cNoa45n9n#lCQV?b{{v;mD8w zOY4vN|AZTMs24`|3B`^XjjZ|%bs^$2n+r8 zR@828QSPx=cqy_!)Bet~UYDLGTjuG@pjUiRe!pi8llLN{j_7>k`p9gz>$|95dH-|a zzx8`A9KKo@-YI!L9Gdie_|d(>aPam=!w*Y78V+0>6n@Y&B<%0^@$mNbAz@#7Sor>= zPlmlVoc&(mh_L6?5#g@ql;S_Hj?S%ZTtTiyi*|WExe=ND+EH{Z(no+Cve@jmGW^N- z!Zz(>+L&^UeY*W#iP)HaVHa6NR!=ZLjlK^Da0o{b!!g8>L<+6QAct6Ozwndu1>xvr z>l|15e(|W0Vf&Pk;iTVAqhkLjL*>FOHtle)wOk&+o%eF7xMyJPPloABAH!?X2$eJ7xcOp=SRB_VGP^wD&@y z>5jhgfAMbqi$2;rp{e_JXzcSFb?dLg@khV$Zr#+!a5G$!W)E)Q7J6|P_wf*q(f4cK z|J?a`=*?;_f7%i=_7?0{D1)*w6L6}I7 zFLP~}OrMJ6VQXxVKs~05OX*K9C1;~WeKVJwk97N!x+k+J>i>QeDu4R>kmXb4_yV2E zNEcaiRsQF&L5!Eh*RVPCJ0|wCLD(3{cJ}>l<3GyFTguBqX)eZ6EXNA8Ikw%g9b_iK zrb)AHy4W~m&UqWzFijWrPax&@X7Nww2eb;WlE!Mp&a-pKs(XFIdU7MGNBYhQS%$d2 zjJkfVZJlcz;M&NBP5iBw`iJd)@A;K-PIkX!FRCbPt;im79}b}RYxd_Mk02I>e{Ssv zSu^jE{*_0e?);+=NBx~g#>F0m1|%`lG11znZNtc98ZfA3opE?#;S_1Z^1WGlU8 z-J_7+^eANLnMIF62Xge2$Q_jSUgwzg$TNHtPK%3;d=$=-=TJT6QMf=}LRE=;4U#Wh z5x$0oi|>b|Ypm}hpQw57aj2EAx+rg>#3O+QB+=tPH*gE?;nusjkB4}Sw%zjYdOG&Y ze?tD#uC3iQ*2+H`yWdwv-`D>vuMgfgKSlmsLyK!@bzEx6q5)SA(*g{id;B|674oSdFz_OQ+{wtY0Cw;K~Pynk{~8Kp;S7VFdpDS9jN{crM>&*xs(EiQ+XIE}M7hYRRsN0$G%z!>45>l#l^M2kL$$>dZ-{Sf)}e=E)L7cND9U4U@s7UL8f zjQ>xN|0&K){EbWq8)f2@uSLj!h2t( zd~27J%6_zm#(&uV$A8WF|4E$2S)9WKT*Cj~{=fYCpNp;y&({C2-Sgu=6O=dSxP}|J zgGKOAld-k*TAGH2&pm2|G2N#i1|KH#!{JeKc{W+9g`6c@ceA!$DddnH-xXO2- zXO#Vp^Tr(LW0C9f?3=s;?cRkG_MmXS@!}_9GNxiWN-+~P1B%$7Mf^|xIqK;NG}Q9X z(Ri7EeVc#%fG>TIf4!f7&3A7>Yq}`3@$1`P_5LA)PIMuQ9Ae7~|FHgbq&4c!S-sQx zcYO$R>G6Yn-@Vod&=(^4i0^xc5Bz{HyzPasSX^}8ieKBOxKo|(T9%4ij`Vf?0-N}* z^UPfkUWM!w{Q_h36U@<1Fsq3F$^RF>7VEJQ)#IFh6#t(dcP({Y{D1m(B)dO2zjN2? z->4L~3wy8+2XF{S5W_LVkwgml@$ieD_f`FG!Zokx|68DM2Jw~pX13^~*{hvK5*f$j zkZ-@W;T77e%5rlL#GOT(W7^RX9gEH;_ZL}2d%ZN?(f{{A|KADq1DeI9(SlY)Yf{ch zC;KhsUmJdb-ZF@dde2_}^h}9%b&7qn=+}_j;5c**(axerTW>fPpB&U<}1@{JZ_1-uoQ>qyEog zX^h5Lj7RU6LYPQa{7}FBuk<5Wn-lFl(fvE?&BSk=#?C^tW_!ADROgnGGcg-;5%s-9 zb?|(8Mmdk>4lJZEMsB3?ho|c~mWu2CinT6e&&%FZaurr%E!JZrdh|>>9dS`+ftBV+F5R&v`tZKe1FID{jJ;TS5~ zl&9}`K85=Khw1-cqyJyKS+CtpkPS%k-y0|B|39VwpMCHr>+77O&2R1OhmOnU2jJoH|il7uh+*-pt}++WfQRIaJdx zke5)k*LClHa@|*iuOWZk@>0J}KFFFO%HKBk4RO>Xu~Yru&qW8g8k|K-*Nx%x&OP}|NZX&HuoR7F86;x{~?W*>*~MDzEdo%J=lko z`sx6gzN-I>J#dJA1Tl2TOSG1`Q~TXD&3$uUbEw{J{f*z_sM^73Gm!CH4kyuFZodVx=QHL;L}B;*TomSGWPd#}T%cb+h|jkCXL% ztiK;%{rxoS?{U}hMUEScp%{*0jK)~Bow5G@qV@OZt-rVTe&+-G@87on{z3ciqj8M& z_Y z&NEwF>_l;xOU_62d&OZPxfoUSx+?if6o;k4%h5nj?v~G5`Se@OP5DRN1LOawrzg-* z;<71CIRO8qMp@7VwDJp1pmGS6yLK-pQ7yZ%2OquW;8k=OQbkc+V&{&3p-Z)i`q{ z$bG0DWqt&C2yxd~cfvb(#=A%+x2WUDBYunFSab}MNTC%O zFgX;%QH=Z?8nVYYWJ*0TmOdU6F&Qo5qCJ47((~hwW9&0bFGY^clEu^U$C@d8kZtS~ z{9E&Xw?6vA{O{TRU!gC((mp|R>E+)l2yJsbe{?MI{LxA8LUxn-0r~mAwdNjlxz@C~ z2mGEiTKpEx|D7+Lg;08-HKY-*Aeiss>_+jJpz3_c&`M!0F_`fUpzgzg0WPY98tJ(zrIfNsK zA#cC38>9Vij|u1fUp{chgAe?Fes9;|5Ba~{{9hbL>Jk5!&zt7^M*HNqUKV%Nd8Ly? z3ayBKOW%_oJu^nys9=Xx>LbkQm&lHC4zkm==NuFDA)F*nqxz^ilstzj{RmI@6}TXL z2@Nax^=ssFoqUpc|G&g>J6#9rrIr8w5j}}3{&NjIxPe>f#a-M-+iw29JV#?2nJU-R z<=WPH{>ZPtcVC*`lm95MXg(uvXmMO?T3jDvztVY(zTfcfqh*+52J!jm`TYl6UlDyU za)t7bE_7n3xY(>e3m251;q++lg<`T?e{a>3@?k6+QoC?+>!u`hrq&CVD^XxL=J4v+2=Z+=&HG`udV9y^npA#{u5iCC;~q zP4Sz;FxPSOu@Kqc6^6y+QY=UQI|A&5?5q0x&~{sYJip@Ueq+(MP}3Lrh4R-*Hu9JA z{qe#n;T~=33h6}O;aEkMf7PD)hK=GkqYT?oi5*|~ zixBNau#28P=zR1&^a}f>Rr=s!$Bw9vlKmi0UCZ=qk~!f6jyZ%Qh~XIG*mLz(i1z=< z{<0`^`(AxgT+dd|i=DHL?eZ2<;tsAU4nI6q91fis9)7gxQ{nK2Plb0*4G%{i*zeMN zwD+}7gzq)o3j5X!4c~unXxM*#aCrN*kJ+R0W8nwy4G!BE{mOUSe--vgs}+^IM}~^M zpA0*uebOG7!@{mXBYdN8M9BCphdn!ngnu|_>}%Rb!*}B6fbjN%A;!LpCngHRS-+jb1zf@vymROIaCA*!xJHllDF{E=;hjBL5Z>Ke z5RMJ|E8ie8_h-Ogg_?1HrEFS<)yKN4KA#WuBVP{5ai2Fv^Lcg47mR;>K{|7`Zemz8Hg{^zoD-nu}=TS2K^gr zio#>Z^ncZJ{EqtPqxPJl7hy1lqRlvd`vP+WkU{54a|6~GHz#u=&3_oFf5iHO<4B?T zl>U+T^pB9Oo6L#0SQLg!rx>F#7A|%6#H+M=;K#kWcGW%cYD9n-f!()j%-|`ym`-$&(i*BhngLm z9;y8sqHnO!7`60PBh{xctR>UrMzUO;(_(#M_Ok_Hvv5ZH9L>Y+df*)F=lnc^JK8?K zZAT?yvkJp5vWl#pS7>b7bNgjMsC?HL;&t`ODQ(_W^^I%Gf8WRT*0`qHDe4>7^@s0| z_+1+v^-=DV#sM6{5&UD&XGI=E9BrlA_&5rrF7BgxuQjD(`h|z| zc$c|%X?p?D`~L6R=?BJN?-_sHZ~PTo)ic}h7C$?++g|35iRR8l-w+x|FT!AS%;V3` z;s$_n^x)L~{?K@rl?8eIU*=71L3QYW)vY=k5PPj}LHe z=bufVi=;NU0SVM=qd&a=*?hlef5>OYLi%D<1ncR4X8t~XIaXj5R%0z{wAZz3oI`&8 zt^SAgj@gLKD8qJCVi)$HjZe|ex5)25bB6!Rr~RY-&zkrj?fj3O{0}s$JN@a*Ka9QE9gyBA9b4sWlrC9~ zbM%UDxW8Yu2N3-duArwznIOC0)qg|weqC80Z$&!p;yxbYF?zmhTx*5nUoHs!$$==s zV3dDa|2jDw#Tbp=*ZYUD7~RF0%8y ze#L12pP!oBWBp&W|If4eOy}*@@A&yxi0r#BhsESl^o}rAU|eQ7J)i%AlCXlF@?2Mu z>0zGl0Bb?$Yq1_3efTHnL>ID%<|ozcm=S8NX|F^4)C~3GjF2E3PRt0&M>E1k|J{r- zY)2(_VGs79Z9%E_uT=Y2s{Jcf|COr$O4Wa*>c7&^xYPQ79Jhu)g=TshEt^V1t7A^O z|FLN^!U1VbM)fP!laohKwZL`Fab1WBA43B@iFzcEAOBk5{XlAOX^1-}ZGCymtkRIA zSA0>sUBDg`&Mb3%^ITtv>l^3#MwNz|dGf!=c@SSI-7V7HDgW)xgOoH{kwFe8aT;fF z4sEY0f9OC4o#;XqIW)c}{}<&S(Ow75_995v%D;QjikSO-K{}Uk1=mnLV21q{XM`Sl ze2{AszCpi*l(6~td+cN!cWa_4| z_Xq58w9s3-#i>I|rBm^ecKRRf$4}4i-__MO%%*3g)v?zcWBPnV-ww;73!U5S`yW0Y zV(O8Fep`%cX)h(0ql#X4QNQ9<`~L{9LNsS4iTYFWN%nkoM(F-UsXcbx@0aZz;h3m= zttHoEBck%TnT*Cid%rOwl+i2E`=!z+`w!~xeQS%pxnQ2xzsScs=0Djxc$Z`LU>^?P z5RM>*V~C^8o&@daIA{-pz4k6>w}%0;=j~yD#(v(PLhlcm8tMHZ)50xetK(`8dw&z& zkt*+Rmv@3p2se<)E%pznXA>alTq(37gB(txMtfe1x^dbcL}SX)-m>}ef8p(pIgPV8 zhYPrbE4YTd|GSs}i)a3?cK&Jm{%rr>+kD+e{9iJCm3?sW1OJ!5-XonGxP@M{v^%EB zoOybNEzyCy^ufpt;}f~Ay#MbyqjDd+uHJlwO^Rygc<8vtsJf?Jy)0kIfB))r<7(R4 z zXhrnxz|qnfi}9F&nlJgOL zLn)d&5ywK|#c1dwUx;j%1oGd%(GKPJzx_AL|5E>1j%e?LRpe@{#d>VSAC-STZs%cT zQ@P9{m;R5-|JF77AF7m}g!-FJx>=g(CiMf#=;gosNZ3wxNH?=z`n#n+N&3hkD*Kgw z-|?;2!Y*lqz$2uBb@Z==46pPAD` zkE7=e@8{e4HrlMi`GtN{aVfMSgB(tx{7c&PuN8)T|JW4yDKSPX{w&TR|NXzO7lupp zE4YRp+`ujL;x6vvAs(Zbzh7QoX#J1(@c(c;`=CEL5Jeb_XwLUgayW`H8qwau(H?VS z>G|=;QPu>|CnBf+Ad4vno`vG$;_4#*x!wNt!WI8TJO8}8pC0Wkm|jv8 zmeVsETpL!*ODGh@*b5xdmuI607}ZE!JZrHlqyNQHeJ9seL#9 zpUfQQ|I@poJf->n`ga@e$UlxFbxQtESX+P=v?BK4g|JIHd$11&P(8@}gJCa(L-c4( zMjejOV@URMZAdsK+QTsaz76p_M(U4^j*pH(`P*y|RD98QVSZ(-OI!vyoWyCIMa@qB z?_TA9KRdmhU4D^W&QESY@}9W?j=6wKxPoXc{xz}(H_)aWwJS#*$bWx>>^h@--cdf$ zxSRhwOZz`hTTeDG)Bck!!mVU9f8~~RdT|%`(K3tu%r1CH&uo+S9N)O5_g&@sg|i2p zWA77NqrbS=5^E00B2@P=UP=x{mAav>F!KBRhT+1+h`#xk6s{)|Y=f=(fwtW?F0Q=v zw%NbHG2Q&N@*f)0_-FeX3uk{*5GInn&xJ5K|GRz{vge!Dbp-PS{)>6Y!l^E{44F>w z@2lAS^x2q;j#t=+=u|el=6N<`?t%V)zb(XKEX8uHz$*0q(sjIE5W4@_dnnv9!akSe zdThjI#3tzjAT%Zpj`~R0MSr@^dk_5p^6P%i7za9G9EhyBsr^Uvjn_Ep z=?O%0%%Z(|NBhrP?>(6-5X~9bCwvGMpL{YupzXZ=@5|aK?N|n#d$mo6wNd-^&F?h- zZI1R2fAsy`JK8_AqOqI*j^hLP?`R&ze@6?w75V)I3iubOe$}}DnGdaBJnEd|T({$LIEm9}S*HCRWgLN?(bjh09Q^`v z+Se?)5cTz65*K4fTp_QadYJZtyn(8fPx=nx+S^;gy=ZvgzM>up)c9{LAH8ml`3K&K zdcJ&uZ19dG$-DmZ;rRjgg&*QEvR~~J+O`@0HtyZA!}#|>{{K#OdzU&sZLEY%+E~K> z?>7GjDdfk$ZyNvRU$!FCPx-oRe*9;n^^eMIq4Fi}>HMDPn>qdei#7-4elH?ht}CDF zhQajwzL~FBPeUJ$26}Ri^0h$uLb15$+nl4xv8X<3T!Ea3Xns&_tvwiyTKnjF>s)u@ zsx^>E-eIdfl8?`h2$LN<71L3QnV5~aXy2j_64Bn<9fe&2!(GRu1{8<;rV2X4~q(gQN^1qSh z07xSzts-d-MqlR~N)AUcMq?~$&ilRq^7857xPix>>eZc>po$zaK&G1hrhr_?6eoS@N#6E zmH*x5AMIEE5!Kc8+M@&-=*d&ohdXW+R%0#JVZXJ`ujZ?o(mk^-qPI-+JHv+wg-s z<~&?~F}ywJQ(^zG7sHRDYg31v_WN0!L-(gY7T&p|egE_)!jVzM;lG_I4u=m85ARGE z?z?ct(Yl6)gVTnFA2tmQ2X+n)Kd2oX_T%jhp9}k@O|<@PV%Xa>F??^z_^>DO7oq3x zio^eHT%h~w_T*Q_E;z^5JM0Q%Yx;*v!dGw&(ev;9Y;owJ-$3`f_G>Vx(DR1xD|DL6 zFRmANQT`4en9RP;rzIbw@BiudkLv#`cro;+4@40LV^FJpIeqBg4)Fos)@SwY zkQlikG;I2oewc5Cn(OS3JM0g{C$K-Jus=%JA7oN^w)E#>J{DpzmSQWmVBl<>hu8MuJ$oJot`TiUEZ2tXu?=bTF4{s_6tE96UYq1_JT?NKy3c^Nu z#&~3h@yN~eGUSG^J*KfiCb31f6@=~LVkHHklH7&r2L)jdxerzJx{C$uj)HJN_z)Vl zm^ZnA9dgP#yA9?azQ#U5-6rk9Wp=?mc7Z(Q_aD5jem~>+A@aA5_j3=u{j?v?2 z+amw{ z$VK5MYx|b;3n%HPkvphwYri#q_WbXx?HA69i=_*~Ir0Lkr}Yb$$SbI#*NyaC@AV7U zgnQ6%-gCX`IU9FKM9;q67#uqweuk}1Ce#V_c)I_`4ga}?UfjifJj7%4UH!rQe?ETH z_n6=RcOxA>GbrphtZU z(b~R>@901Habd`hpH0$#zr%S@?3maldrFaGQGL%|o8&}9eGpOKLmZQZry~Ddfi3cQ zf_;Ro?6z&}x3|dNe_}hb3wnNA5W1g>_WdXby?>>?_;i7NtDX z4r}NqxCcW*e*Wu5|EU&Nwcq@Edc2$eYYu)HeLIpj{a0M#p0rRYF15q_|H%Iz;hEBR ziQ9wpUcUcs^Z(8D-zR(k*-82(@O1uv4L>@+|G^yBH%pq!r2UF>Ai04L?mtHm!!g8> zL<+5FJL>+RqtN~B=l+g#fBU$<$glq{bbrsdzn9(Li|%ie`-_%Z_t*boC)`)*9Gfnt`aY*e2&mO#!&)ga>r zlbEy#;Kvx6O4HzZ5R?(!W7@(8NZ(R|@2^Z8@K$Ivj0U5NTg z>_jpyE}A!xBzr#of*su0A#Xe%Qc?ITe;!)N46^Ew-p{MQ=qFMB1>t)0Q>JMjknf+a z)jnuDpM9VHsQq(pYG2TCM*DI~`*KzLazgupT#4`QN$bP@>1pA!IETB@`=!08Mco|z z(5PRceZ$jvXf+A#Ut0UurTru8g%kYr2KGvl&wfEVmr$XPF#4w075X*w;0D^(7~k)w zf1r>40c~@qb~$RBv*V2Kqj8)51svDLrdH}-(Eg@h)xUsfen4sp-&{JqxQqM9`~PFu zT=b{@|6_XJHOgs~yt_wTO;7y){^DZm%yl4(5c&Uu$)Tuv^$-64aN%M^-!6>4VVL*- z@3H^Ku>Yrc&g}9ySwF#Z-TR02KcoF;EXHFZCgZIxedgGTHa>rQg8g5`{$IoX-@yK7 zhh@o}@V~kKr;G2K<^$K-lT&)rQHt`d`mjg%z6QPGcW)SvGqzo?ya><7LM+BoEXN8& z^G{ZhtMPsB-deIn`HSYCtfyzTDBl}=|C+uTxoOJx6y&vGfK=OgQ>7KH^Ul~XCr@kR7{s>~&TBs~fc~bu!6OJQ^il56*(C*VS z$l)YT<1EhM0xsbSt|9skK>2Up2$8?tBi!@(zTpOWEBXz0k$TNKTk;_uqwoJE|F80~ z#P_G?*I$e>Z;W1qXX}4F_o)AUu(+WZj;J0eCacKmggWA@Z-k1iuG=-#-S&Q5_kMJH zKW_35)g4KUb$mBJp}e3hMC%87{`t?s_$co41z{rD`?k3`-i7EJY0+Gj$>OGBdi2|G zl<(bdhW7n$vT@!FnW{JSX}zg_dQ<)Mruym4(75u=(6r{waD3C7A+_$!(7fbLed}+A z7RR-w-wZX2-VC*`z8UHeM?Df~KoX_SlX72XM$kuNHs)eJYNT0b@Gx6J!rrvJT6 z|9e?j=$OS=ise{=RalL+XxmZ7{x4(ym+5~m)Bj$k|Eo;@S6Rp{EA#%B>3=WN|6Zp5 zy-fdmnf~`O{qJS^-^D#B3+tt`5t~tlmO=73?9H&9o|)p_ppw1|xqj{qx*XFv=FR-K zOJdIV^gDFb?#CX-Wxr=Hk7a+z*FJFv(C~`;g8Y4QYz@73f#c}$E#lqpg!|p_!1KMM zy;i0wzo1R<-y?|O7~)9ckM^J8C;Z9$AN>kVhxz|xY99Z8k#}>M_KA%4pGjTa5>nD> zMFu&v+};w>I2pw)(Lb<&|4cs{{eDm#v6uZzM&GhHCoXnyOSnK@LUrwyaD}{vDtg@z zw*5%Hw(t!!Tx8o{WoPtb>yx*{_2Mq>Al}EkC_}F{r-AE==p*<$@K28S`$T1#dMUSM&C!R{`$Hq*V*N|$b@jT zHaTi%qp|auj+>3Sn2&{6jHPHRH2-6ezDGP;UwzU4u4>=OM)g1wj&E|W*SY^o+<&x8 zbN{D^n_$l#>8!vitj5#%Lu=`g&EA2H^v(F!>-RtW{cCMlD058jZ^}Y%r}vlM{ZaM8 zKjXi@$)6|-l~LT6-weCRp3l4)`jYDr=PTABKd+5UY8Rt##Z;&>D_>Gi@37yF> zuSxwl;RE|`q&aNPk^AnSD_?vCbh2BXjc+8_8-DNkcAv1GjQYrdY<*8lX&_GY3{Pn^RbKEh+kwglu$bP#pw5?J8`zfR9foJ}o za8@`+HeNRViQ|q-t#tplxc_7eS{K>>-(0NjukiVsqdvg?Pv-v(V85vUGR~31X`ID5 z{6E@*3uG_Z`%UHZ8}8q)o)0l~@+I+An?4e*kk?SXPQ6RsK(v;k4v7bBh&$e`n`{X3 zq~8|e7J6|P_wf*qQL~7Dyo3+RSB)Fju21ki_wzxGTPN@FANKNZaU3Z;^KYB@w`ddB zewcskKan5aXN9^gW zuYs+%pY-Ft1aFS`6Jq6C3F4*|IR(m9`yY8d;UAzAJ6{T{%@Xp zS~&mw9haK$LMWBaOw7hyw7eq!@;INKQC2&I7t$9aC!AG=yU5Na^NPjAUbB9jT!HF- zFN9U(YE;Q*op4+p)(Wpj!*$PIxc(kHlKjK^dw#V3top-jYq9~!G`rn@HlqyNQHfpH zgMDaYSN?zP@6h-j|G$d=&!11BnVwE4-)#I=BGNxP1@pGksIJSi+eVHxl+9m#p5JSqeWbLgSo`?bGU#@ zi0xE%$f|?ZKhdwz%Z-cnkaauM54Y71527-pexN6>n^)|&TgZN?Unu{wHTm?rxQ~Z; zjJ|*4{l-9ic>YY0aQ7#z@A>)*zW)jJ*ido9QH;^Z@6Y_reqk(qJSJi?df#{<^wxNP zHh33>r=t`z(K5+9Hl$COP0x(<{?O;r=Ofq6H+-ZHMJEP}TZqM2ikRzNPOiWz)a=$i zYv1Y)YoD%bpB}Lh&TF5LL}QcwgLeG~bNJF|Uc$dN|E~qn_jR9r|7V)^8Ts%3jMHuo z(>^2G|9`dfti^h4MD;rR-z@T7IeMHO5Uoutqi28YozyNjbUhivuP3)I&~`8J{&){l z+WQ?kK+;>Be}nlD{!|-mN}D;QPx!IE?W2PV!n5_CZ0cvwZYxk^tA$+&}Z^CZ+Thsn)*p8InTT!v@Z$st5zYXOB{%hE==5ND~ z^uHX^u4Md{!%1|1&;Eu>l$+;16}~?}-|Nah52wZV{?#X}4g0fj0B>*k^RR!)z)-vU z6QO3uCqlf>N7dDz2#Iwc3-#%bg=G8ZLPMX=hND}2htGdwM_&v-sWmR!^r`Ufz2b1} z`tVRqFaJkA`j$Z*5& z`Sra+ygzKeDF3&_^`f%P`2Ncy!+rWgJVxL3$}t9_2!k;c!%>XU=-FyLGTHrObMDAq za|b7qlQ9+3QHq(Ejk$=v?>C?9(f_|N3hSd?9EIzR|F8q1zVYb#dPf*L7QY_*eK=z-oGnd!J^{tffb54m+@(z7e^0@8;zveUP2^yno|HhP(2%!f%^V zhU$YOL)BjUV$$Q*1JZe$1E-tmEIP4=2pk<}Ch~yEZ#djEA z%^)Ux4B0Wp?TzVoPBM;({T)BVHo zQ6oaiI?(2kBh;@W)UPA-rHu$RuX=u{Lmc&&JwG&@@%&B{hk>rK2!k;c!%>XU7>n_k zh{(r}Ef^Ul)2lX(4AuKT5~k9pqZH59=ikx(KGOc))Bbw*XZx-Dx7s(d=Z!x3Z*gqX z-xbZrpD!-I|G>KiVIh4nmSQlPZa#py1KnSdM}$-R&9NYlA&w+k+MT1x zIxPCr{g?CKXVjOG+w0oo_v!x2GaVCKVDDBkh3aVBmpqB84f^eO8^1N)ds_G`8ibSD z<9af||KGaP+(5i_Q2Vd_Jm;9F;|CY$mv9CD-ul0qSG2$Lp0x3Qvj4!!KMU9Vw+A)+kaE>}M%(4fpFPbmLCHqY8|Gcg-;F&_)D7;T%pf9QCRzj%sG zca?2NW)aQxZd8Xf;rJZoAJ6)q=&e!Q5Z{lH&T_24Dzr@V{!Fru4?QzW`+>Fe^~fF8 z|A#JgV#gcSFllQx`fW3+x9HC&x1(x%$n7kV3_`3-yg9_>MR zfUK-x-*?)NntlW^977yQq|k~CayW_8==p{^oQ&Gvb7a&Pc!9ixE4YRp+`ui=yr%r4 zZjfVXlfrEa-2Wx+|EtPBI)^F$c;^5ATjhV9@{d;Z z{45%;GuHoi`j9$(Z(5q=pR|`>8^8YLX#I0h=>NCgOB5lR`!kpv8tLD*Mc%SoCZLwb39@c#wHx<)SikP+e_4};{KWsfX znH*<+zWCXw6OQjT=is`1`h@3WAr_-WT>9P%VJSUxl>ct*dpUgta#j3yes)(A|DCKE z!v7w_Pvr;4$@&uhH`ySZB=i2^2lme@|6h%@C^r_fo@{eX?dX`Me*vBJE@U4W=f7wD z!7~20bdF0e`qp5x^wXn^ztdZX8Go-T3LB+U@shoCzHI+YdbPMJ>*&kq@osDA4;F>( z^hzYJTZiz#+W9-y(388wrEapB$$el^LlA*8RnF8R#J=Mmu;o~{e*Jm@-6<2q_x zTmJhC)8yZ^Cl<-S>yG9}9P^(zl1QNy8RT#hZTGx?%6A7c=tT5wmuQd29G-oDVSxOj zxm){>7PM-gVu`|VS~_QO4i`{;(f<4Vh)eYNBXc+I6^1MHYeH9PG8wci*Y;`#>ajrT?K z@*B?c^^o^}zu?#(PY%7(xr_UFh{tGKXZ`^?kU{4~^`AO0OXk#xjcNT4(mcLR9|4*d zxIRQ?#Dos8~-dK2V*F*pY0chlf@W~o-eZR$?lh|4IpF6)I@SJreZp( z)u~b4S4xl8K1XYyXVPaQdC~nt!ZG#e@!oX@eE`q#|FQMa2ki;Ff3D-_V=-^OG_jSnRu>N78|CC`nDzOWDunz~&#?Nlo-gXpffBR{FM{0k`EOJxyziVTg zaGXz`a%?kyF>U@^3tADIVEjNjM-am?REvwgYZs?S-yEt#lAc0xKmWg$|F0jQp8RnA z(CYULawz|*ckJsSoTQ&d#jmx)U)JYCKZgsrge&MNChrEGX=*3;!$3r|u z-@g-YoT)!K5UJbt<{<}TD25~Y?m@cSK0NfyZesu#O&^QgE6S$!BASEUN!E;E|Hx|{ z`E2}S0Uv*k`_DGo>K%S+UyS$vi75Z9G|T~X`ieB*$jd zFZu5elUYRHACA7Cz1?{#u?u^UFTdB#IiNo+zX#}tkQ^rOgIohyKZ^bGn&*T3{7kF-*)eXdyE0Kk{MKd&3*rhHNNzdIF0i1l5mzhhYRR6HjsV&<;e`bjfFV!@2ifaE;3o&SA!eoudC3`7wIqs_YZ_Ny~O2bq~u>YXmtzi)j# zJ@-nf@i*(2aa`IdG~cm)87bdrP)v_Ma%~t* zAB*G^*M?}1f_jV>mpWV;CX$oUQdJtJlGBk+l!lJiN<+pz1Es<vG5i$4eS^vR0>%_%Xk8=IvtP`il z<+V;e%Rl43BYDF0yI+ZJ*MD&q9hV}Tm$?74+<(uum8{vJeC=1h5U=u1vmp}~ zz2oPVZya%s7>*(Lng7uzP9~8;E7}$*UrUthHOkjYHan6#6|1>E|8Z{ zt!!Q)uc6BS>ZB7d(as9rK*JsN2I?J`KuxXuqpsa?sHZ2;FwL{VR`ul5`uCRq_afWY zFWe^|qV1;o7aeS_%no(>VP$KZXN_E%{WXXGwTS<-`^lKkgfE(#fxX_(i|a z_rJOaU(>#jy+4}~29n)m5!v&3dlaZc21jx1hM{E3mXCztWHHhkjMI{-mHYs5Jo5bz zuez?QuIr-fyzDyF`xE^(88P#_?>hff`gD|{+Pv;6ZNW@>^ey(fLi3F2a}j-$r$KuX zeXG5moG)%67Go*4e$bZ|)rZT4v#%S|CG+Fo#&cHDS7R;Kqo&>bKh&`w;%tZd0orzU zL&GF({_OWZ<4@?g%>O;lrg^}&LG~c~4~@&%AEVeGNR_ZZ zq?txbANB`QJAC6nItOqFN6;dV(Hi|2J#$_AeNg{9J&xRM^B2%{)BJ^d+Guepv?3Gz zj*~cz-tQHJvt;)#^ox)^ztLYZPJ0pgGse5J`m3Vv^j`8?zW(EXT%lJ_VgIeg7;Tl3b8L>HG^F2p#eL{i-ab~AcG9j zWry2Af*meFf*meFf&}};4t9_rK`ee0RrWDSrk_3!%!^!8=O&5L6kCZO)Fx-?lo zS^pcR(5GPr;uwU)F!!g@8S(6(Fx&A~-}YQ`9u{B` z7Go)v_umM%rijKAN|3=T5%qc6n<*_&ivGA_^e_a00{8{<8kZtxp zo{YbW`z%y{-o6Wi^figE`iFzUc5)|nV=wmOAnFj?HaDT~z9sdJ#3@!=Y9iL;AqrFw5}?oWf~j`G1+r;Bc0n zyKn8n+xB>)UqtaazO{bkymPv7Sy-L%{a49eH0&B2u9G)W@46=OB=Yunb9@IaeA*P> zw%K_}G}4bO6zQs?&KvZBTI;>^aIo~*cu467`=4x&bA*p}oa@hr$2j^J6zK)zwOe$fwEynQ-p5z% zW1ugx=J&?e|5D$Nb0=X6s=uy}mYji}U+eEDdq3~FlXb6GgxTa=%)zi9mu0clKFwTjMd%&!7ya5GzG3h&P^POVn zmH)?j@Pk8@-?k#opRXmiBaV+R&7Wsab~=t@$m7_bI0mR$S&_uSLxaM{zKUN#(xMuSn(h9^;L$qm%S8rzxh)5{?r*^*PH(- zd~f}K3Y+#Q4-Y6059$B+AANtXzWj}fu%F)hynXLH%Y*cuU;BRlQF$pp{lxP=bmb%A z$DP`8j;O=?z4~Y6A$v%B!S9X7|4q4a-5MUk^KUAXlr=?uN`ambc3xQ++X}jrMO&_z ze^@rd+z3AF&y1%TQWqw7QQ=mO4zY^ zivHATVfUqHLgDAng?7LHplz_VH7ml~>j#EC%LaxY^_Ld)@tog^I04NQ;T`p)Q}mwC zSA^4KoZEkvJdcaGjH_sSdAL5T;URtGX4%oQ-4hTOvu3e(+W8tSO?A30bW5fLs#-ACV zzeGF#LjK(w{JZtq_0d94Ei=~7wKvg+J1F_vZQd8|IqrC?f9OQl0euMi3A#JY4MTws zTSV)%{-N!b`FCXcq5a?J8NOxqg8kp}_J5;45YHo2n`78-qi^uO`p515VE{eh{+eE} zuF?ati60=a+UqQX6X`=ei9QB`9)qlL7mNs<@%y!OP%)dXIR{NqY- zHe4$Y^(V{2Dtcm$G`+4}iM|f0j5G^NqFKLD&6ka@_^^HI{FdIP?VpT$h{wIdx6*5| z9sj%fLGwc8$P)F9Wy%#avhPQPmB!=ESN~|Xj;`x>V=v zkcp4$bel#RKPs$YhxrS*Oi%Qe7F?xYM~dEpC+&YVANTxzJ~G_&TOaP=9v;MF^L`(Z zJzujv;kVt-HwJ_Ool-LJ7fj$YvyXu|DzrqHf zZod9U|1rgH(=Y=KZ|eWn&tFMT>=_lB-WnBV(PtwyRo^a>+eU?EY-0w`LL4g{e}MHr2g+S{^OGV?{nh3Jxbg9sL(QVR7g#= zw*2oxSmoT+Sc7%gfGTW7$2tA~SM>ki(*J)=|9_AEe-uzetNHqE^GAiFNaNUT{r||K z9gX_)kL>XO(vpa48jwlHEuH>9Gb+^jkL}or-Pnu$IEXrQxTZ6ypTB-o$i-)>ZWo_z zX?+)U!q#*CA4ieKF>z+{qe7N!cU^k$sBl<338aulR#*n@^xQCUBS$YryqCh8qqMP( z3fi_4|e{mFP{CWF( z`WE{no|`CaDi3{Rb~69J=WoItdTtp1ANS}F5Zmku$jd|BcqFWjzuwOji~(p^Yb_wM z0`>L(vm6=fcsQQ)D>T0>ooKuy{V(W$MFP!8q6I08aLs6pL9G9bBPU=IrlDhp`oBEb zg;>w(&U^oB<*|+YEVA_)|Nj>M|33f!Hvj*Uyos#**`Cq%|1s-Kh^P0H_P8K7*9SKq z!Yuk0vef_a2lL<^&&50}K;M@~g+=6IEXB6p|DkL-xe~o!(I-I0a%?pj*A-bquEQF6 zz50*lE7GeF*T0!UZbdD&Bihi6@SXJB5w_(JADO-vMK+{>d_7ykw)7k`{!4i8H_O98 za+7p#=G$%==DT!U%Ku+d&e9V|A&tI4<)NLd6Mv2@;sj11uHj!V{4_nTN!a9m&eG2# zHIn~}VJcG`>XOxkS|5g6K!v1H62|+H2)ab=$Z{SWz3`Ut9lrfjAanG3p+g3q#i5@C-)UYlXfNtI#xE z`J`NUvTkt8diN)kApMWW z<^MCviId`27Bs&r{qCbQzQ}Xh>)ib~h&mib0x1+eH7InPl>hlyPse}X_FPdK|GiK7 z@0)0QRsJW_Y{M}!<2bI-*ZxrcH@`nEu3omjoy?(#6F7zHMr-GjXK@~d){1bEjCF&{ zo`fqwfds{OHb8r z`o8_k>hrJuH*Jp2%@_mFbxykgO6BOk#d2A@zzXdIeE;LJ`KXOYSzrOqgn)%X6 zv>=6*;#q~&Sc7%gfGTW72S2p4$1_3>-Pb&u8=jHA{UZOob-wWjIEpw<;MhYxt8=oq zj6Wdjv{BTGXFGOcHyYZ^|L5E8r63Mo-{lecU-{7PIET`@4}0YQr0=v|{=a5@25mgCjvD9bmD<;4`u?ZNUrUS;am@*w z!fAZ4efO;6j)~U4cW#%wnM3yl<4dj>UqTk;tJd-Czu%9FGff_Ier6c|V5ISBzgr`LSETziV~0X^>Tq5Hcgf8KI`WSmDB`}qp`FSmX7VOi|= zyY8Aad6VozL0MGlf7BNd%cVFMInGJGZXapne48%_xvz^_Yig%bwlhk>wk7) zH}<08RrY_5aRKziA-?fz9|{NQbx7@czyJRTUs=Dw2mAjI$MfSki2Fwrkfyhz)c^2Z zeFXF(PT&+854@i~Haa0K&C*($-*7?t&ZvVp=Pb_SA}-@9dT|{c%Jj~d&yYjhQ#CI? zKJ~xj^V#vf|Gh%_ztOuP;~0&0GCffnzIgg@2ltSjuZ}R*`UCWw=g`H@Kce^B>fR5D zyVHNPMH{Z}AgoTEU?5q6hGEtpAcv#0{=iJ>8f=`H%K2|FIT*$ zn z!fbTRf4~1xUqTMuL*(<3;vXj8KcoJ^2l?QBKc)SE-u$2ak8{N_53$TxKxS_B3ya9b z$j($BA(tcfoHDLV+4hVwjx6StapX$BtwQzBAC?t<_Q$f-^fg$AK6YpW8Tfd-LuRkg)2+Oa1 zRMx%zQCZ`|_sWipeXp!(;(KKYvU%ovWl6Hdaf&Rws7ueN{z^ns|rP;_j5 zRMv^EeUHj==zdGw8UNAtsH~WLRMxuWy|T7zkIIhTdQ_I~c~o|co}p(iJt}KQUHzl7 z;o=#A(HMgUVfEq|M^C)q-o!hBJ_)Ju?oC*7s(YROUfC33=^O8rO(SO@yXU>KN^%x5 z!n@kuE6a7hS2o-6T$J{OQ3iDHdao>x#*xx5txeLJkoIOWIp2REg?X-7fJIo0rC5%Y zScQ&D${%#yQvURa7kLy=MC)Yfcb`Yg-YZMvm^`0R24uye_3##UK6&9`Su-}>m%p*))eyFQCpQ_79Q&PyPLG^&l2Z6|Xm zqC@+CC*mBy9J;mX=TV>+(JEiJDFcq4WDAn|A3L=NB1>2j`A2=}^z0IK9NeSl<|sG#Di7$7 zP*kQAkoO$B=c{jQ^BeN-fzcSV734TfK;Lil01GIvJ(;sU$H zo)yTV<5sq}P25Kh@UzI$`giPRw6X1|8#zFK&hN^0i)%0Tqu~|f|ECTJ2kD9VY}_36 z6#8M5#vkrs+g@hlg#XJc_sb5}-!FS-*Tb>{uih{F(L!bQl80qK?0>s#pR(Ww_5ZbO z@3PxvZ@+Q3Y|qrYW#8|);lKV{+4mOyu58zm8)a>6BSUNb$dG>Pxp4Glb8E^z5suw` zE+pcAA&o6Z^ylzr43=MRU($&ZGnGsDzvhJzl^xQ409rEAG zs;B>#vTZy5OW8lI|8Hg6dv2G-_2*9c{h#kYEZeC}`R>gBTDD`^e=Ym|ocGFhA9#;V z9~5f7HYog{?SGW*z285){o4O1+k+ozzu13VJ<7FbaUK`(jylq1vN0Lg0My5DM*q_l zeF|jqp+1JY`c+0+H_$n~xQ?6X!yVki19Ys_PqR_~(=+;Q(CwOhnf@mfXX>YUQ$G!k zB8_A7_2VFm_E&|yQXU?Or(ccsLkvWAV|mC3ub}5L<=S@2!%+Hg6!(;Ce=O%asUu;8 zu()^KXmSi1rfVZ4C!k(<)9vz*z$C|UFT<8|GaUD-KU9)Ee>A37UY!-s`Oo@>$hnw@xcA@!GD9vR7bBZ7&XQb?+{?yUEX;y6h*@6z5f ztRie4SrN9ppf87cjjRM;r=jA;e|3BpY`42Bhzw|awl>UX%zeM`A8B{lhaMpF_aS@ks z6}`BQ4tBqjKi`GY`i>d)e=YkT|A%ayD*f6Djy@y(IL7wJy&y7qsfZqqA0&(y7ns3P~zbg;J z9go0hjKMfez$BFBWw^)EI+SC*7wK<5V4p-X>$)8< zRzvu1WcH|gyro?RHI1(SwfC`vFCblU@3uIGvQBw9)VW)+7yHrhn*Mj?<3V~t8QFw7 z`eCFtE1UNzqjxE*$%L@jhmeXAS$aE}L+nH7V$+ht~rgfIFF0CjH~Fy)AfHI@?-C^|F_tG_Pg*Z`%kuxW&bC#|73b5 z`%h*ZXUWp~KX3f$`ajpj*&wVQH|dER?0?UI&_^$5Q%yN;ahxQZ`NwyJ$9a#X`31)1 zXTAS>!X6-_ZKms0-^Ezt?;Q90jyUMWiN2G`z7w=2wGW@vK73qT0FKc!XT)<`{zONa z{EaT;@aOxt4G`x*RA4CT*oocPi~Tr=I`n;0|DQZ?L|$l;rkK{|bJB(u#PQFEotHog zX|yAUB2J*=b?*mV&v-w0I{%Me?CEt@nPBrMEBB{{M5n|4-HZJG29Q z>qC#*v!*_-f1eWnX`Dsj*CWG4@-q6q>{!8bUpjd~|Mxll#3#*f7sow3z$5f~OFan# zQGt$S`rpxo9J-N50Y$Xx>u&QOM|bG!CXeYa&yZQi?PU77QDLZfhGPUqBfHQ!`l!dy zbNZ;e^m&h?Pe9Re;Who=`n0>rNy6%uj0#i8X=oUtKb5RR9K+BgoVTe@pWbX>LN|UE!C6qlKP2KI%`_|CuGu8f#nBe#6=-^tt$}>;Js+f9U_Y zFVDNbdE!}sMOciibY-M(DLwa=Z)>xC0O%`GbX;&8=lFH+(yl9PHR{&d$APSWZd6!D zZa{n%%e}KIdg8b~?svU+GVU=D=N88~##{YXi|yEn-KhSUwf)KcIEXqNMgl3cBZtD* zANwSw`7i6ubqVE<_rLpw{OR|;L8HS7vZr}eI7RmUepEP3p2c}w#ARGXFRr8Si{=YP zi96{1DL;F6;AEDnr`35l%aZjNNawvvl1hT?13#|)C&%N-t-dp&wdb#7`c;A7r z{8aVe>H7bs%l~NFA^&fa|JlzZ*@EaJm)3u}Ypg9MxW2}i?%MMHVG?}`I=tUbWqcQz zTjHI}dEfra<3*I_|JSqsN%{W}`%f>;&-Fj;8;uJU#|%_r7G`5E=ArL9=6|a{^?Y$u zSm3z#i}sQr7h@@wqwW>uC%Fo%u?7vo>XmEj=m~keX^wr>=v7FKmo{O^snROlrTul; zh9&8l5_u6`;fTyVn4FZi~HXlq~~5xhj315{NFcyS7V(&-G5B> zUu2zg>ipMXGJ!aDAVsE8+W*dPaSVOC;~ZKx`hA&nlCclH@s{}q>M2d)OQ3m|aI_$W zqH9jz6i(wT&f_93qhp)&?{JS~+^evg9`^<+sNa;{|842VQKU~w|8Z$U7VSuPD#yjs zi|e?F>=5TXXWe&tj?M1E9r``QHT?_7JEwcBev+5<69|8VhB+T9)x|f-i*-HrMjU28 z9bqx8anHjRRDW8TBM-%Q-Gss~tbf1V^V(zF44JuP{wp~YS@y4HlkxPQ(YAI{T@p{$ z)hZ5?k1*VMBhdFHaeq#pqmRKjOu!^e!8FW3Z11ildq3g7zs`sBZ6?ve2B&?>dCcokM-9X6oPGpi!EqV9_IG0B?W#(4+! zAE!6$GyZRvaf|c>f4m91>3b3TRAXA>n8#-97j_VJIE(~RNTam>mo*T2zUy0Y9QXgp zkwu)qDWuimPLpSG9v4xX|F=*33jN9azpM1?c)EV*0c8~$`}2$Vg-s**pTqb?e9k0V z_{b@&am|)9e0;w0CO&ho7%e_X5opPzYTs{fcQ{b*h&?pOQ=;#<Tr&Za zFa^^v1C^MCQu^=yS^9Tr-#`(q(%E)R`aPGpR>EKO|9AfB{s*(g*>F<&rEM-fprC?3wq0%AM_{d->s$>KRG?DA)B_zkGFiY_kFu~vi@D~m&`T!8Q+zyTIakC zsKQp%Vmo$XH}<0Mlj^IVG@gOp^Y@;8MxKA&9Fa14{uyzTag0xDvj3pRb%72#w+@Gq zKniKJqw|_?5IJ-qk8TuEKpR`vs(eY0mFJNe&jt&NV-K<@jsH>3HQklI!LeUd9jU*% z5#s#6oc}1|6i(wT&f_93qqP3dOwaU|bFX>E4?SzL;J8S(PE`M!%D;0?nmp$CgX{ln z5+29=#r;qF>@`r+YW~8nw0nxXPuoCg|37U5y^iC2i0fqZcW%aG?FDgvrat-|+`|K8 zPx_B5d`^0<&AlD*zIVt!D6RkFIDg2!k|&%qP>vBkOeSAtE65bPQQ^0t7>>Gg z)B!5jd3uZA@}~6hP?3=1r!e$f2sYxZHazt=cUK$ zx1Oorn#`hom@sKsC7#t-gLQ~)7;*o<4fNbT_IHo<2kBc;WLFFK+27mjFIg+BZmls4 z;qW5;u#^%!Rto4)4qRgA+#OxERxc^MB0(@ ze2(FsI3C~;`u(%SF2)90e^ThqckazWHT#vdRTqvoqG$zPRWDSbIsqVH{E=gC-JtR~~wz%}HS8qErLdJ=i`0^r)L&i-$z?BvX21Q* zvd@HrYd;g-IsWNzATvGusD7$7o6P-M|EaL=&B@^hFH8)3w@nUjzwu(&v*N|@{q>&= zyXz;0@6G>2*hOy}{L#?b|Dz#oUeeKpAJsPUv2ZN&v9R5L?Zj^E#eN*bPtJYTIIXhq zB!C4~GVQVModag~m7Zf1S~ff#kVoANPM< z7!q18J!{>Dv0>}PvEi^d5~w-;VyGQ5F;tHnAGQq}AO30lMEgp=7*c*q?N<`CK`9Wq?JC zN27Sf-uB2}v)4Vw2pfmGGs1sf9_o+J3=jIK<5%&@*qWieI0lMz1D`1}A7NBvW^@of%D6Kz~R4!$VKgeq* zB2$;NZzv0%^lvP3{$eaet+irm-YE|q%Dzr?O;uhh)Ba@qHlx!XC7he;u!nHOm=B3yR?M;U&v0W2P}`*B84>C zkwXzDQ1bsbvymwIzi+XPC>-MdqE(!2gZaPx`MS6GzjxWzp7;I#E5eTOQzt%t#+ydk zGhLin;c<-oX?pHmakH~$>D9&;l>C1-J3pQMWn1Ik;dSD@Bms|a1>Wa^LpvuDE$~AtZt#Y7&!)U?c{Of1jM$1ICe6DNsgzWWv2AM;5|>5PBgO3 zM|=-W)n@q$aZkKFf49))U7__g zb#NSYURr(p*hBs}nML~r`HNlNj4dc=Ygi=CI0s@exfG>wC4VrkguW81uo`Pn&Hs%3 z2J7e>P=&3?-uEB3)i>!m{&^R+(|4k{%)PC5kLn(!|nm&63+#2Q2TEEX)J#n7irZ=Sru{=$=uI6Xj1->{qw9|9wbI*N2-xU1> zPT@4p;yf)E|7=w<`Rp|2y-2|93P0cL)D>8~-=MzYyn8*A2%Ae6am*wBwEg{Qo5X zx1Rsc-|y!8=a=yR=kWhu=Kr7J|6bt#@@Gr?--$Dm=l_%KuB#(s9v&mEahQOH!TjH6 zJ`yI;6GQmF^wRtnW3W=|-P@b)v5fyKe2R0XVFuEficm>r+(Xtq%yRFu9M49VvHm%9 z>)U^Nf9|pIbv`JX#F;>xQ;r*?hRG+tw{BEZ%8_)dF)62 z#6H#feLHrdTAiTnndH13f9_sIW9uwTu}gJ>h-7}3;2I46yE z&#VK@bl>d**W5}UoXWx~_ze@=_% zEY9O1o{XQnOpj$s7rS|t-iy+{p8MqAUH*%_E-bG9bCc{tgSLY^6w9#^vHfpb z`M)zp&o|;Yt`SgLKZc!*<;!Ye>B-vp$?9L2(??bzJ6;`w@3oa4*CdH+K-AK=qqP2) zH0F=X|77D03y+^6~t~oAkt8_jW^H0R0Y9SKJ$th;uRS2}=*t z2S7eTcBXf>TU`Pf|Jij(y#xatSK#URqm90Swch_p@Bdxz|C0BweW%$skVK1bDE1W% zb$z-W9o`nlvdOhCsi^S(+hdyOq30+@@5;3IxnsMc}$rS_XfG&hxeeA!A>nP0?zG(%!`;0Ze>4hH} z14k}K^(TyhBWu5EJkocc56kH*u?nlP2I~;}=1Sua&gm<2T!pOr+e+3VcU&6pO4|_m z=909Lh3Cq`cE9b!ZtTT=97G-Z^ywcad-w?nve!C(DKd?As;Q~62nMas7j_YRDE%*Qp*8h4EhN38eXbZBHqp!wWL-`2>`a$#ghy*n~2aLox^ z$4&I%4({OrI@X*2j;=H2sGrm)e?{LsSwQjbsL(3*-!}|Su%-_Fj{=ReC$wvL% zN6<7;|GmES=CS(k=`CbRoR8dNzqjQF3`7Nb<&&Z0aHOw|4kO6X7=v-h-tr%qK+jE; zwik>upie>ZO=;7wp7)>K{$tDQ+8+*$4x3){{mnQ3{>bPs&3Q9WPj6b`zFbq9$MrkU zUb^GD2F=Udr{~Z(+5MyGW#^$ZFW+-%aZL)9;+cinn2UK>fJIo0j`8}R(KTH@pX%G3 zC|{pbmqwiH^>qDz?>miS%KQwn&l!J$w0FEzJj<~XtB@6z!D@PLuXo!17klRsDkA2Sx>~y>vE%X$g^!t_k>mj~>?Dc!!FZ79#2T}b!ZDKF_ zzfWqD_?31~VKvXPx4-18&{Ie+Q->hikwXz#_4>HR;E5Qfz7hKZPti}u-+Me8`7M9@ z@%*(E}O!3CDB4p9XH0Y9^erQUl|no{XkiYfv7--x_&3RE~)>YQzyT#ZjQn} z_5VHWzdC-Ky8h8v*Ow2Ekr~Hvj841rHZRl$Fvo8z*m8R8cNi+(;mFcU{{L?tw=HzN zC2syetoN7v0LRg1AK~257=v+$b9d`8fu3-0P5hEc^eIR=ZgHF>o852kuh`gT{`)(g zA*MN}+S&j!$V$w@Y|O#l!*Y(x> zXYBw#9UN9UUX3+ahc0!m96IXzhfZ@J3hG`(bkpH_`cf79I{F0pcs!=@U7j~3Hm7%-{4yd%dh2YBj)?MTl}I}|D!kD_((YI zytAmMH)$`3edy;MUqlN%?ql#|o`2&q=@(DaM(2^u;!oa{{)f&ZYm7Uq^&T&~_9}XD z9XHX3KUx2gU5$NC`IGV*O8I|>^zV`W1JWtIf3g3r|F|Qbdw76H$gY+CYsNh6mHzqC zKj$N10DT~e`^3G=f4n07!YhQ;wONOO9FB&`9|I4fw6T*NJLG@9co%Z@@@8Hh zUCX{~WM9bE8}dJn%IoRd^8Y1y9NA9!KO^ray|)benAaD1$Xep!%#d*{zj^eyf4~B= z>jm+n8~O3dmm$gk)ZH@o|7Yc4+p*PQk@FU#L0CPO(i3k;8*13s+Sq=i3~E7gg|rG^ z?wsCW49b_jW-XhFu+s4=ti~D~I?^xv_@Q|s$NPn!3|0@(2HnTE?^PzP^ZN!=p}M?0 z6tI;Z_x!3Qw__)I)mL_tJzrGEA@}1T>hQt-g~N^$NMYx$5uxuL^FMyAZ2G0VBCPlK z+L*{9YCfi2=X>R0-?ic4huD9^T7%OEh64|+!+3jOI4D2*KL)A*<9A7wK?8me!6ENtul zu~2jEW1;r;N5cufox+yM&xftUo)7OFDG#R|pT&7x#AW>GT)B1H%EQ~%_tbG8e|25?g4($rzy5qXCZ-t-i(|_0bt?<)b-wKEKeKR!lYz#*>uMdrfz8Mm4 zeIqoLeZx2V_0asHXu}zv(TWDXUVxS}?1T79>p1kV z8%Oy6Y)Mm||Ibfs9;^LtzV<()u*fxwu@uWu{aI@`ldG^A9rgVGBp-py<@p3;-f@8} z`Tvvo|2WDYP2(6J`pNz@?FaZ3;#h|bNKfw{s>qBwc~-q-E4>!m(RE+^=;nv#5$Dhr zSMV$R*07nqe`w4yy>0~?`lWA$nqPe@)V}jAZCKK>Oq%>x9NQb$@Q8bT?|1G&{I&Ie zH*eD>h&tzPxy1(|6G$PAKPmsUv;Ecef7SUuSpRDmM-D}tK+mVtN4{cCz$ev5ey-m3 z@r~h>@IHN4eIL^&Mn8-5xQO%;&*Jmn3YY0u(Tg~)DXs;6ot~5axHij8dLQDRc;VaD@BenFq$leAH)hdiBehxjcS(Qd+xmjO9p(y~ zhXq)K#aN2vSc%elOWLJkJ!O^S)u`52RA@Dqlirv#{>S@h!qf3* +B!%m~rZcuJ) zopU#!3R_W&?dagcb#5~TW{2@VWcLC79a(T}K5uAUZ~PCAIxl_Am>0fZhRmY3VKK5dJii`mhtP9x@(o{ChqRAi%`eIykH>sX{!;25PwLg}JEtlkTdjf2#S1+QkjKBTNw?p>)(l{P+4f?9T9oCUe%AAC6 zX9K+orS)A7sNbo7#`Rsc3aeXaO=ogD8ea9j$=#@bMf-|pw1B=B`*9F;IE(}u_wg&x zbc-E%$ZlNYSI|rA|6F0;*r2u~`$opG?8oTQc9r(`dDXfyYmLv_sO_E1&Cx!Tb9VfW&E&LJTH2~ zGulX9cN+EdrvC1Gu<-+q&!dH&y35WFlMcU^=Ep5`9ujX!Uz_l}^c`^?PPpbGF5@bC zaUD0&hmJY?f5b7hIXs>Jp^hEP+E&l#>GRp)`FNHYWa;hOJTt%F5zjq5z$3)EL1wyk z{Qc7Zy7WJ1ogex@6qm67$iF82ZyF;YtWJDG$>C^NZ%hC=8uc5c%eje(=5aV4hZb?A z&^%Q-M@s*N_xt~SuL(3yl`m(iBa^9#>gQ+p=&{D zJ3fNa`c|Eu3%%6-V_s}(m_9Yu^l_H8++059KU^{^#{g2_T#%RJ@(rRXZZ0a z`R#A?4{dAt@kqbIZ(qW1pK1Mpsr>ly{X>sAkv(5z??1}kcbmWAS#LTL!hUgYnXX^b zngIvtwcoM6f^`B8(-Y|Xg*mC_4%UtLo*bvqjvN{uKCTZH>2XcNrgMK2PS8&wwNZTt zNi-kV{?E5Q!bfe|#!uv<#x>(T6*Ta|1HCN6+a8?;7%v@PIxA z#n<@%eBzS-&wqU+td1Yt?;Yhl8t4Pb3e?k^7E0F=e$-FEu7hb^aRL zq#sRZoOe>Vdrgupi2i?R{?~2&!5HrPIOewYnDsX4qw#e8N9Ao!p6^y3=O41~`{e&U z^8XI$f0_M78pn`9oG09lwEGw%o^hCfNyrMzxbG?SoVsW9|EJMspy;@OytunjDXh+O zm_^P;1AQ(z5A{Rd_y3P;vvRx$E%Z3|ulXGNfYSK4nS5mVJs}S_@00)0LQl!Vi(Ru6 z%drxxuo`Qy4joDU{|fbwH`G7K?v3gnWWjNfY<*Y$$I&D5qV|Dfd3o>9WB@7NNz`UGkZnW{#IErs66bZ@5O!`L|nJ4jy#M6Qi$`0<2>RtJ@=~e z0qs#oajf!T`up`4=`IR8fm1k*zAsgTv*dYPMDO=L5-yWH*4?}sk8vG0(T7diI5taj z+>4|7WBu*1q0Lly;iJ~ICwu?3UwB0JXov21!1wd7>M38cwmp3yDlinoF#@A82IJ7z zZ2bT4%I)Xzkubq=oc}k8oPueXfw*RBB{>VTF&7Q5$^Y|}o%Fb-K@%3x7a=uP{+}-Y z^9h<~%K!V`x9>@IBCq^o$CLCHGPZx;aqqEjrnEoy@d076IF=&L{aj9Vu;ZO5t+UJS zceDBVH`!?RzNpQ=Re9QmqsrDa+xpc0t=ImK*hjrmJgX4Lz14h*o&QJgZsv!=YGI{u zZfn&AUSapJ)%~myo)Z?^#p_mlD6Dh50ae(F*alvYT6(m7P1sJ~iRjaRaQ)!Mh0_0; zbDSUBQJY_vezbTlDeQL5UhKy~)Zs7^NTK6C{~vLWzuY!9`2gEY7EsKnAKj3C9Cc0_ z$LN_W?Eky$KjNIow0PQ)LlN0{4&VGl40G>YI7L4le}7Kgj`Qe7>{qHgr2ezaJ3<^g zaMpQo|Ka+2>3UtdP^}H~qVN`aoD29QohAQwhi_yL``_uCP#-9b{~=Rn;`~4T%wMq< zR*dt@{lZnU7uRtU9c$(Pjq(_olh3;^$y+GUi^tjjJpW((N4N0{AK5a-+;h8s-?<R{;;bUK8JD${tVP2O@4m;_ z1bV_fHLbPJ3w<|IGx?TCE@T7HcuW7k{_>{j`k&D}UcWS24(Na0W!-<*?8SZ@L>;!= zmuCH^n;u&K`iTC2bj{I!kM7s>-@l^&{#E_=Wb1A7|L>dsf81Pu9P8Bok8D!^f4%Vm zL-gm=NIoK`>81T=+3H#JtgsAb({tmM4Q%dQ`aBe$lYb}5zv$M6us~Sd4&!ym#fW=nEG3sC z?xE474xhNC&C&5HwA^?<|K3$kp%<$Ah1J5>U>!D~3R_W&54Im{cf1pOQT=J-hYq%?6Mtd|aU+>){y(4nd4P`C1Jd~e5p*`?Z)(iiQu@2&@LdQw@|BU=k=B~*9 z^rz!H4yb=$(f&7<|KWR1zrg=M23fSDG_Fm3aI3g#Q9Q}VMxOnT-Mtk`{@VKvRS)pLAY>)T;qw(%ddQSWOEY73nlOw`ASG0Y7+}dw1Si@1h z_P|tO6Gw#o_Qm?q{n26H^wHsmZKK29vVUg|+t9E_pV`~<|1Rv-Z}$Dk!^5t&;o*DZ zSB0JPSA|~V2dc}B(|_K0l)>KDRPSr1y@34JWn4w!S6}t}$oEDx>mU%i;&w@S>ymVMH&E0`O`e(JwFa=;pJd(6=u%RipUKjv4q zAcfYA{9C^I(f)k-iTbyv^IcI|$Ibaq*9SPR?-JdeeD?!<_&xgf`Nrv)*8lgPb1@GK zkZlu(-xtwyH~9bi^uvFLU*Nkh^3My%UlP}C`vN$pZv9|(KwW8*dC7eCrNWk@{v7|; zxrwB8{2i}Ci?~u~&Pa#*INE=zeyLA~);B&S9aF<<=d8i8vQKKK86VcsH=qhzQH$-^ ziQV|gD{&1i`$W(Wy;>F;wEG+``%L(0S$U}6s9&w@vmx=y%+NSzdN^`zx-xfiNNt=P znwOZ%R`x0NFYOQRInHrNp!qrXg_gnYbE0|j;ys8u97X~uq|uI!k=kE8=Pu{w$nKe* zjXWRwc#3DVKP>cqmPjXAdOuS=TjxD_Kk3bbLry$JoWf~j_u2o!-i(DWmWLd@i(d1c z@=)8YPTwQkdVu-+_J4HkmN{(C$-&l*)lNunu=eO#ahyl}1>X%4!Y(?#jF!w`<=WuT z?3!fW{*UVaM_%E7EKyHisN7huUXD`x!(03f*Yx5#ZX)ho(?{OHJ#;vybA>v_8~hLQ zFZLhigS6o&(l~|;vS`NxaXdo5pU77jh&XPig6y@fLE)?AVJLk#;@F=NN z&&FKL!vZYAV)T8-n1vU$&ka*wa=h*56T)(GB^st1{~&*_q9;yzwpdMHgA~05N#%R= z^%@7Wk8Dz@op>Sp!+s^v(#cL?ifh(k1FEnUwb+iG=y;WVUdTSrXP;kZpZWCp73}jH z>@(T=F8hz8SNORX_`he^|F!Jj@Qw1?yn1__Wu{<|JeTj z4EtNg|0Uy|n{~VRVJ|DQ(2zIg+I0z(*8iEqt|H|)jh0vGchv*P?J1&q#b-;FU=1{~5oWkj-`=-v4jT1fJsh%%l zKYa5!|BDu+5ZnLHJMSVc<0`U~-NSTcK0P-~`f;6p6U8^wCDc3eYo&j^c>%&=`~MyC z9vXx{ARnRB{y*RUqFzL) zhrinXKUx06(W&yE`ud-@|8J6SH}g%l@KNFzzzO2aT#!HcE|chS{Ld8f$-LcezCi4I zETD*5_f_+i&xWJwCDWWY1HB(E50zxk?>-b}k$piOkDQBnSb#-XjHOtPl~{$|AM+>3 zo<{w%WZ&=2jUMUSLY$wvPS^%ip%#TN%naMfx z5y#;t@5*=TBq=iPF;wH4+GFO3JLe$ka5$cizQ6ab$ev&NPRQOt@(h_n5hoDm|D7UF z<1Ef2?kyD8c)dt3t@kqC_%-@fJY9d_Ip6Zw*#2vN`P7%(+e_-HFNJ2u$!#x%7Nk1O zH+D@guHz;?IRCHDaffp{(KV-%|6j@fuMBw z3?K(0>%S|=p~!SrYL{|f8!N+b$0Ja1Pw~D=_qovh?Qwqx+}{!R=Rc1fa(~F89Ua%C z@0RqDxrgS@)ALu%pTA)Kyk|ICoMSK!6EF!w-S-r78XA{)zAt>>`401Z#nW=v^A%T} zcxE`S608_ppQkX|yfK-3iA~{6*ofpTJ$2#XCzr`^Ii^-)Z&HsDW zxYIcQFBpd^Y!y~x4c1`;{$l>0I)4(SbvoFID(7rPEw&@p|96tRu@@a9*?y0 zE$x3eI$r+AF}5j#EWKU3Ky3fpFP?*_!(n7!<6mgoOVD$#vY$xN(}{+Jr^(Ig;hQF_OaDV*4*&%=}X ze+%eO=Kn3CFUJ4h`G4o7ahB^!^>1T&meQAFC05~oIsb1L8-*hNZ|4847S9^2!v-|G z{&@ah75&NlzpeCI{Q3O9?ZVR94tA2ek-a@2>?QXj!>{Y|TioB~pyN6e`20n5pHxpF zJ6_j*fv&f--=mu?%_EMnE6raVqJA(^{eTTlJMS2qn?aV|zKs2x!@e*4K>aD!E7;(c zH_`lDqW6&Ei`Y*Qme|o~bG&$!4{S;CU#f>DI zHyaNoEYANsOP)t;ySPYR#s}yBU3J`xznK4*|3AjL*aW&HTT94e}2Lq5=);AJ6|AN`Er{W;lHW{(S!5<~QVTjP~1>{@OkNaZngX zpMbu&w!Zp7&-3OOI_~}BfG~xehMHgTzpbtJ*T($iDm(ATzrT!eah({XkjiybdT3q6G=>ukq5 z%W~mQ>Xj?$tFRjD5ZC|PKvrQZYOx(Vu^WBAwND~h>VNnLZTj?sF?^G@N3zE|I863_ z$@7geXjh5hUmYCMWaFFKAJBxv5%<-}Hlu}}LUproI?l_Xh!Z%4(>RNcH$LDWstPzI#Wd(Dmt8~ zxWyT_xZ%VNSDaMij5BH|K@vul*0iQIZBtF-9rNqXuRC{s-}fz@IC0_@_l*uU&Y0mu z#h6a2RKf4_+z6l6ZJ*B{ugCj+&UIbq`gzX#yw9&|r}o!6b#pTI4~|yqKX|*j1lDJW zbNgfc<6os2$Nki+v&6oOZwSXW{BM!}9(Df;i}jC%e=%-GIKTb?`~M?)N?Ef&rt6HK zu2H9^{~W)>M1}GhQ~ZVe_!KgCijQO4Jh}&s|3-dXTqzsFrt}wCo5r@KoyuplS6UB& zOlTACXf6y_&{HRkJtLoqXX6^gwSCj#pHH70WLw*;Nke}D^8XKRVq+6*Z5R9LTwQC} zI9!~6PtW_QQ?weX?C<(|6lwc(|gn>;`;ylKVvMsaDVJOP^U;r zzgPdnpNWhA70R#w|19q{?SDw({u1eod*8~YSE_#t{|&MmJ$FoP@gB%d*VU!YkYA^# zf=#bx+tD{J|09+k?{VDU;_vWb+<}kc0LHp}e~4pDCoxqkU1aGw7aPY;{wV7B zFZ>+8#IF(C-M~bDcuXJQc$MeB!}G_~D$gHT#5Dr)>wnM4|Fi5rIebjn zTJ8B`6mx(2oAG}G$^hg4UL?(zqVZXt&znobE9kf4)%XKsfAO5~dUD~X+J3$p=cgLy zBmBqs6Z{!2%wPTsdXGFG=V1RO{jbq~nr#t||2~g%use5~e_pdXbnCVHKlkrB=5O#G{4M?tAI2T{D8>%z|Hnj&esp~^Q~LHZ$kyrquhIXnzkBeQ{(mwx ztN))&3y+Ya_Dh}iZ+hu`40q#RjFj33_tPg=NLxSpgY-`$C!7_|kW>19;{Si!2Umx^ zmEOxv?SECP!+z^OV^9?PLn0_y^xs zJK34`d>W*4NIGP%XOr|?`p!znwGW6NY<7Q0CEQ=ewLj_o=D&{$-v{DaOut?Jrq;Tj zOO;*Dm5b-a$Qo&sSu>wLS?&IJ8H+&wXT&ju7yo~-!~Yq?wUgfFxTz^^3m7%#vf7#(1!-J*z(>7lfTDD#l zK2`M0@M+id@TzBrR{eGl)v7zREeVgb>W^x^CVb|gu@iqU-(TvwZu4)hW4j8f3Pszwd$GSZhU-D|Ks+n!ae2sA2&Zc?Eg|>*dHzl_f_aK#3w7B z9hx^@9UiFj?Xt%&koJpEudLWxSr8to(yr)x$bN}WBL1a#1#ZQw@dsG=|5k_BlXKVl zM+7;4oAE{DpWx5%7nt0oepV%a)BhT|Qe_r0C2W1QGUt>voyETge~Z7vi0es<|1iD1 z$p3%HJ8%FWMF;zy5dSfH&p~}yYwytS$Ag$)LnkqX46?|vrJd|$*EsuE!~Qq0sjX})JKm>_zQ2Uu?$~yG z4n62c3Zt0BQ|1>GO21V4&YeXL1NtNeF;uIs0mJks&rfJ~uAFrK0sn}9LPs6{yO>W+ z?@78g{44z%i2nrW6OLmfd+{xC|BiY5Cob9tw(m#63-~$m{~y+`^h^4$v9w3tLgyjl ze_GW4kE@@?e7UIpkGRG}Tz@*&|DR^REAR|F6VJvqcs|CC>HnY6|BuNN`v1@B|8HjF zTJ@EX16$SqF;vC=W0)S-h8wZp=xN6lE(?<@mW46zV?4eWdJa>_VCazNrmi!P_8h&R zxTeym`ql8F=ZJ1~te#5majSWh=$rN4j>P^I?{UW5M%VZPya>OCm*N$;6|Y8W+B{Y!u{|vvq{z!iOBO7>e{A1d@8~gn=5}EV1FRnk5pMT%$ z-*xmSuRroX{(pE-{U7~^>+k)IwBCci#oyt>xC0->*ar3=6Nqi?|84%?wE91W5bt$( zmFJHUjABmT{sHOC$F_=BUJ*VgILfH@MG;SpN{%p8T(Iu4n62c zYTP-`RWpYi_RuuqyC;fH^V-l)y+!0)Jo@uw&G`%}v13_m=}AJ37U zRs3IV2i@pFFOuj(KmJ`B^Y~Bv2n+Zx{2af;SONbZ6PPTOev$Ocq+cG>_x-z;LH_;_ zUq7{t|6j@f-|qV8qnr8v>&zjL&eC4_5l=(M8veidE9gD^{ci0U&!9gOaqK``8!$=6 zf5<&s+%5nl^Z&qpjq1Y(daZRu|3myS{sjNy{J(nnA2|$E$^RHSrF>RC49oXvas>JL zf9&s{N#`%{m-uUp_>R(p>;`?Zm3_f`=zojfp8wbIi_^xQS%c2D4K;1v z`FGVACs=2kAQ|WMC#Q{{L_h9!&U|kRuQLLK&hh;2-G!h=0OABjq0dmHY<2g?~p}+a<1XHcy}A?@!cN*Ny%o3f@F6Qq5OaQ2nfFeDpaEWhG+ z=`TR%M&&xXis zCE-uVKSP4v-F1Fm|6d6ICHh)iH}d}(XuBl*wYa~*d+@jTJA4>-;G?+EzjlBgzmboT zcO&~B1>s(@@w>()@%!(mKZsAG9iPL(z_QRo#{S!Wa{kYkg%ml8N#xMk@*DeK(hCfqJ$>{+>c^jtvp_OWWdqrV~em)e+n2 z`SqMVtG%`Ipt2;|(qzK7h#q<@6MpTuOZuGWCH^}^9(b01{1^TzTtR;Zo{4AU8ayAr zix*%%m;*!3eYYU|9yyUV?k8g|BK;MJYy0LfmEhx(ul9YX>~pJauf|A|amnQCk+x4K zTiAu}D%Xbmx@0x3xtjgo?VfHs8?vAMb@)U3|1tgqe}=!nUt;Wx^$(D5=h-SxZ z%j3w^@&Ak2|5Em!OdVzak1Ja-LLbHabNT=30P+83^G{#?l<|XqE&ad2d+>MoFydN5 zcaR^A`Wt*7A3qxo(A!&<7X~pjdrtj_4=2quMw-rrQKVKuI-{6G4kIhXF6!tk%cGstJ+*|-Ls>pXvSm%Gj~*X3T4Ri3}=>v#Rnx6kk51$YsD4==?l za4ViN|Crsm*#1}I`igyjN7#S&I7Fsq<$p5&e^GL@wjjJ(8h?P-qka5r_(L)=eKvH= zoDKW`WodYuaMl={*ap~B=-crvC&@nHelm{vi({|aSMX=I8-HH1G`zzx??lHcdHVCq ze1B|n5gUzU0UKS)k0kpXGcad-5QY%v@(uIL)5iGZ*9R~6Z=XWr3mi8=PC9Oi%m`=6 zTqcBfInV8QA3lJkp8to)J=llNMfHO-&fVo2&^zuL&`0l2IOk@0#C~_;F5H6);}7nm zUl_ml0R3U)wki`G6YJ1Z^3NmUI?#;u_hE18*)2U#Bm;@?7{PIrr!1!gt6x|L=R`_whsg1S$68G4gRd zfpZw~jm9+qmkhZ6a^EhN(=S7Aj%{$R3+o@|$NyLGanW;(Pdmf6o#o?_{YRIFFH3Kw zy)ybmA1hp-3x(1<&67Z!fDEZjqOp3(lX!Z}uX{_CZ=QQB*rgWgZx zXP*b~Fdo5$ezNF4bO?`~;D4U-j>*Zh@;{jo&XTz*`QLLK)Q>WR6owK1%N+kPIEv0` z_K!V^d9z2Jig`0h_MyMbxBFM}y7LTS1QW>OQG5|&Yvs@No-a8mzfO_)^||D^90s<@ z-)rOv@hRJf$+Z1P$WiA?H5h~Qr6u9Z(u#8czevd2$a;g6MhopJZ zH7P4H98((nX%4oe2T*WsmMIT`EUmyvO-Reg=`tuBO>!dK!d#J;`kGPa#uE&M#h zKEB3R7KH=W6P!Hl9eF>Qgm*OV-N?6D`gQiZ0XJg5_K%y$oAGiie5D|4BIjPXJiLmG zeIwETe+~V0cq86~c6Ir;kZ(nt`}a2T9Z1l-tJN!O3&J~v--Wzi$u9J&TPD}(e=-)l z%h>U5^1rSB*Ss=Z?U>u~K70Tl!oF6|0DCc}AAg)Lo$ueT;(u@De=sG{Pf27s?|2qD^Hh`{K{{IU8KYB~}|AqX2etbWZa#rGekr_1>N8~FdYM_LQkLcEWB z01x94bf6o37{Ul9ki~pwQFxS``?x;j!z;rV>0icI@imMbagWn0!`JDPM~%%F{wDo9 z$Z1c=ipw}|ii~sjzbF3t_#u9RjzaAZt5$}`=y6U!xA5ci=!^DOKXv@U-U{zR*|%?p z^6!YU-8YjaM~;PX&M}Q~{NL~R_KxxoFm>GXL6)Ag-vztewL)EZT>3@w1(N7Rob$V6 zDC}L1%diqx;wofcY|V>x@_vaj0AmI6r~H>i26290obwyk{*P;b#FM82vcg?kXofcwaN|NicR@BsZ`Jc4}rd`8}vCo{;e51BDv#I|nqA^U}rFhq`E z0t?!FvSeKU|50-O#lAK2%lInxzj#^R$I6cf)UFig0Det;oCDaw27H~~_)6`IpWp|( zwpb4I?vn4(w?%tEx%@wC{1=<>KlZ=3E)VaL#y9aD?0>QP2l;*c5I@1#A>-$o^>f>I zlFa}AHqBmVj9()M*vr91_6I5C$N!5Tv3->6JouaS_nP?W$gk_GO`wmRkA3ZrImhF8 z0_U(K}ZK?d$V>Rqe{p&y*B~wnNr*sZxKj{YKn`n=!JJ zjjdT8UQVB6YbQM4P4rhGSM1p!!?wry`?3CUcgwHBYixTX-h_Pp!?Sq{z2~HNGiQ83 z{6D}${OOtVb+hCNbp_A%;oU33r`_*EMb>^iwIV!VZq9=ni>wLx)6k*}y>D(Y+<$U0 zeDcU?eRBV$ukPpJ-o?}56KBtarc-B(iT$sr_2<1%AJCPdx8=&v zo>(0oL7TRSCzNNOE4kX(ldD6*|KdK|dQJF{H1}X18gVD?!acYTsS`JZAq*y86b4S* z7#-axPeh~|= z^}YS6wLjzd0OS7VjboY*zJsS57w{4L|7>;mvi-h_k)6^!t-eH$|5*Au`AvKW`F5Hr z<#C1bn4DDZPLSWT?fZBgCh$Z21driyJb`oQOj;ixZ5?;?tkwTruHU#+|F^#5{u=$? zMf$(j>;JYbr7wKAS>JW5{(W*(cr0yvKt>-nIVpasLEn9)^#RQ9&ylIp(y(M$p2lTZ zi4ps!9e*W#@~C4^t`1kxuSTv`f4M&73^_He|9_owud38H#SSV1JI<^QtL^tZBy8_a z>bG|t*9ph}hxL`Y?sE4b{Uq6`&$)}<<#XBZ>-2EN1-;vef2FKlq#yJ1*-38$$ z`ptMbHeqbL`uAb>6_c1k=BRReQ5_Eh?qyKg`SAhD;EUtG*oRT!?)9E)S!sBcv|fWg z;r{05_zwMtVQs1Q9ep(Z(eOIk-iVG>rQuEFTaYj*B~`gPyp{eoyaVsVNQvi%chM)w ziH)nn?ezB{H|N^ep^R&rDp(ypAnrrhgLeOc*heO2O8Nh#p^<(k?n3uz*D0>|nCmL% z6O^edtmnV(JpbI|7|8vBPh;QvaNxb_XTMq&9;QEn4s@drLm0sX;@I^p`6#}KFXOBD z8Wz5NP53$)$MAiVoPSP1_zw9!d>=o=WR356mpX_1@)O~lvL%ZQrsRv5=N=RPIG(^c zjMT|5NBAA@EmZ&I|G1YpH>Zc}bx(1v(7tB(msUpkrk14T`{U>P3BI8Iy+nPnb{qNY5`v2@Vtd5n&2z|88KZmY2*5M__J-uK4ewOcCnpdN7?0h@G z{+C@6RtrB5*CA`J@eSm{SB#DLlJOI#mxLRIZ^F%ZIXaX_i4#l0CVHGx8OQ&;ivAir zW&Xhy?SDJ?pSZA&crQJP3+sqq=9t&vjd&B@g16#rcn8MV^l?mJ68ZIS2H7~|Fu+a? zvQtB3>MZ+DriDk2v;S;a96$I@>AVZK<9)a=e((eI3-A2>-n$u z{K>ec(l9+;>iHLY{=VtPy`IZ|ED4WEtKGPd$H~N?@tt^rehy1UJcColhn_IzBaZzb z`_MmQ9XoaS<+kttf;_clX}F9Y`w~`?S0awJ%OQ@n%OK{}NnB;y)mV*=gX;hF>izU~ z-|BVb4Y(2ghrBZ+(TltvpIjOqkdJS&{bsxz4a4PO6ZtB<2Cu^#5y#Nx`}dSjZxYT9 zEDdiV54=qI^zF;T+vxAWJMk{uj)gy89^OZO0NEdyoBPS-;Y0K}W0Cfd`_PCxk*X*N zcaittK0JUpwl7^<5FVyqXn%Wz-htd%Wy5J@M4@+uZgG7W!v0rY9!AIsWU(NxKT6JB zyF7f6oaf6me!~0u)#c%owvYeUKC8|zE+PF6=lLf6JNO>r+8;gWJ*fVV zKG)apI)CW>zSZ{c(~UnA-(BIk zVgI$-a>VzQd2Z#N<7Uqhog1~kpu6h)e!bF8qOZyOK_h!}V1Ed&a_npHI=m5Y!dvjv z`6s{S-yhcg;aeE=Eew&VZR#JDzRm6W|Hx6{3D0a&yT}+hzU4RVHlEECJyYYk6?jG% z*x)&0gg#p2Ij-^?(cR*@?b~_6ebDij!xQv# z=oju2PLjP9%fpf}`4X35B`*3$8$-BK_$pkD{P?%$YU80l57*%abh5!+Y3&o4-;6I< zp>2Ny{~!J9jl1t$8g8`TO}H5^$0ocAufdphuJM`w%lJ2K1A}Y%|I$j8^Z&_oG5^1m z|9?7!*GXf}xa&8P^FJ#LZzA798tGa7L!1+N(f+gj?dj#=F5B|+|K3m#?xA;V(Em`N?VsMW!}DKd9UJGF}7XXpL+KHy8gzD`tPj%Z!)!;|68k`VE+;N zDEV^V)HkK`9mKlq_sGNs<5}>1`Va9FbeGDX#jdYN{!GiugZy*#fV}^)g8eHA;W7I? zj&sO9{g$v~+;jc>E#ZLgZaMujti+YL3RmO*HUDq5eV&Kwa070{O}H5^$5YE+`Rlmn zdPM$0wDtdO{@-tvzjL3}HYA<-Kk{Ady(R2_j_1Ai*6=EEufgl^M!X4c!CUb*#IgVH zAnWV2!&is!PI~MUx}AJq-2R3TK0wB{$`6ro{?i_E9~vWndaHWGEg{uf9-i>L-X-o% z+=Y=D|8Gy1hkNLgXZ*i?u{_*Ie*n3XP0HU*A!DB@JS?vLczJk)?1(g#hiqxd4ejIZKr7~68I z_80T~&E=omZC=0m{u%TAvt&+q;P9&?}*Rt>(ZOsyD5B!oWH(2 ze2;9@#(Q9ye*R0#!}o=Mh@apwJdRY?t>K&G6F7$@6YjIkIm9ogPj1;1CU6;jC35sE zGVXf{?YnLbSBk$1S7S9gcH9~g;-5$FY4B`t9sLIM)8pC<$!55`zdkZ@1%EMd9JAIGs)=YXEJ$Xo;NB8kheq6qyAbCU-9vUNQ@Y$kx9h+7|C9C3 zB^<}d+-IK$@Gu@h2fDF;U|HxR$EMx?arf`~CNb6I{#)IDv->~j{-rhOUWbq>bpOh+ z^rHL6DCRzRYnXpYob#~B|HG~4pe&2)e-BAJt|2x;X7Om0&s`S2NG|-_rm*mpP2tP* zuOj6+=jT7V|M>5LuZjCQ()0<>Xfmx{Ap9LqdTk;3V5q&j#^tV_*0{Gyk%ce@Uin8)nbqytg1E(tf$d(Nnv zEb_Z9H4f+_>g31OO{UdR$lke2!p-(coz~V*Zo)`|zFP7%NP8~%brE+Lgx3kb5!s#W zg78$0{6}{3%e&TlzOJ*!b;deXa`X8*R)5X%@FvH+1#iXM@D98a@51dEV^7AVIe|FV zbc&4WW=ox?nEfe=?Jw$|C)7XLqhT^FJVK7zubrR&KIwb_+1JY7)}`w>tp0brFnmZ{ z&pLVIXkpkx--mwTzBOzx*}K7aC+<$%g?sSi@d@_{?^jMdOy#2FOTj+@ES6eQr_MtG&7&u}x=XjA!@fK4X6WXSDJE(Iuhb zlb44p#V@?}+2N|V{n{nrYI6R)SBKT)^Kcy&I@zQjs8@c*dXSf!tAB7wxIuhE{=SjC z2_52YCSQ&?2cp}xC#$@-3T2yc{Kr%0PJUYr|C2v|@&9*eejGnNuK$0TV>jVdcnw~M zN>pJx#`LR?@8-AccR+^C>d#e-A#%|K>Z_*)Q*Zx4ADqcu(cZ z&>(KV@e_T*{fqolGPz}C`1F=b!^1@@LMt9>d0Kd|rYt2uj%I=NQAL?Y(JL z=+$S|U+_EL->NWprdS(mv3As5eD&Kxy5QaF*tRqKT4-Mt72zMUbr2p@xaMZSAxNGnw;qDDLhmV)v6b`Mw zDcrO6=5X)yrD6Z?D?8Yq`--0y?)SVuxn2KXrT#zP*MkRFhH1yn;5cUSP`&wm76@{U###&&Q&5C1`N667bzTb@YDQyVdmDU;DWloBHljg4U?Y7R~ zs-jTu9Q%}ax=QkWL??t#;sNc(Pia3{V~vDLYp~9k!*|43I_z>xN1gJ&wlM6b_ngr_&{i0#={4wQ z7y72zY_fN*Fw}}mRb3M5$a+M-x`8~1=x0x?VE1uIxCvSIG>0i}@5Z{azw4x0O zbfHt9^zV1^?=iHMe~;lR{yj$MqZq3-|6q&j*z7vCJ1;%6 z&G-N^x6$~&Qufw)29d@%I;!~J>~MzOvyE-WVfqpDtK0M;iC#SU|HDUZKZgCEVt@Hv z)AYumzxE3SVTOJjvzWsPoJ8k#_P<#>FnSW&XVuM7Whe+<{2Z<`+3#r`9%zf~liRmkprPADcv zDt$j?OF{{KvR2-#UJ^>_YmoD8WpQEs%C+L+`upq1_2^i&By1qdkZ@eLbYi`Fqi{L; z&age`wOw|(wsKR>>4!1Mg%(y-mO9oUIo*o|t; zzk5ljA?K9EwdBI@ED3dFJsNNjhtPy(%>9)8eF@vkU!H%~^3Wo#6>Ui1V*f!&2wlQS z?4K$P3!kzEhrYSE{+4P-IPh=!OW*yRkhcB8`dj1l$zz@$GW5g9(X%m(DGYi4sh#T4 zv7T0`-dv~N-0D9;WE?{>amW}e_2B&ffg6lNI-(3xADdD?&WJxE&7(MmY0Tg_W-*8S zf4wvMnF{11vQOQ;|B!FD#W&on{##udPT21xPGJ!pyGlc1M`<`s?>VIJA7|)i(NFI~ z61^zM$^SE}Lm{~WBl^vY$W=(|Kc8sR?~h{P5@ZvrL#|8Ty}tGgHcN95rMB%~radRV zZ^ZS@)`;s|qkkU%?f(P?`sXo#K@1^<;X-}iD~vzTw?A5D{6UfN2gRg9Fn>20n9ozw2`xci$k{s6rd#vh;${it+I z6}Dpsc48NHqZ(tg@<02QA76Gzx-IJexX`{ku)*`e5KE|JkPxs@Xm*Xhj?1`dbO| zz~9TeKVBBP=t&GBjm`%3|K{^!`g#)VVn$!zjQamHJBe}oWN;Wqa1_TdjTwwp+XoYv z#FVz+j507w=9C))Y4v|;4(VS^VYov3BStWaxi{;7@4e>y_?qjjKk)XY;kb0?E)8Lp zT=;~2$P+k;Q&_~p8?FhFxc=5@;WIdk!q58#ij|)=OTr5Jh4r`gUn}o^m;IfQ|8e2} zn`6dyj2m|`p`JI1DbGGLt1L9uB6qYj4EUA@F=QM^3d8jL_nj#XqiJ=+F5eHj$9+HO zr6>-)jpg#470pU5KDlmGwFFP4Q>^kOu*<^r+=rC5WtScmoKtY)9kjUMzWqvE=` z`E_*3{QCYU`2VMrW90Bz2#PPBJd^3Hr zKpAMCE%dF(31>^#=2iT6a+|nzb&(3P60x12irkLGqPk03J3^)~>=53Gz6$-;h-()l z5&xYV+m7@9H<~L5yX@1k)A~PkmxSH)o_hYjed78**;y!M$D zWD!=O7$qph8mz@Stj7i{yn+8vsr^qL?p8_k@I|1NUj-Rd9Lv)^4y!fxTl+l&jaj$k#tqfWajYUpuJV|T@pP)n~vTvM)ZQ5}=) z-R%Aw`2UCa$z=ZjfAl2Zzc0;C=CfBgrXCGAh(l;XGg>f~)W2`v3CB#5Q~HcEb$oO( z$G;!w;{S^uvOPr(+dr=DpCA8E#{d7dN~aA8bYZ02K83D>KFOb-*ewlu8o3I6qr3S3 zJDiJ*c{l$5FW(PD@2GV>_RAolf4zI1`zq7_ZteQR;`-LRuVVf0WOA+k_g43xuwT-C zWUuQ?YB%Vcwcj!S9nfZQ;I9h85yu|IF-&6y$1zsx{bS;o{CC9rpY{I9eE*NUHz2*i zL+-!9y>9jXtGs`V&_|JW9kbGz!^p;!;RJaSsq&TK6dC(!r`%uctBwEtTNF1J^M&Uy z;Tg2=Tp3P_JA<<*c+~qBpQu?G3hDXz1JYeVFG9cT>q8Q8EXOKw#aQ^4D?$lbiZxh^ zby$z=ZA(KLxe;@3=8KZ^f3-4fCZp}I$3FFd{U4X7f6C9a-oqB#wqhG9P>CvR#}4eo zF6>4%YEX+h)T03h(P?Z@7rM0(_Mn%ZM4$2@&QBU(!v~T7zi1u%UaagWW#7rsB6AY# zcX9sUDP;zqDBnJKQvCpjoTCZNNR?}+A=B%%FKGX7rDyf?C&-C$?Gu@wxZ=Mug-1zd7;*PT~~W7ng)Z@-)uiEILlg|8q-1!9RHaHS#|S=__!t{-*t} zS9#aDj{QM*`ES~vwWr6v{k~d$0@|b(+t@C&4-`pbzwcocS&Xrx>^q{3p2QSAgDjhw z!@vrD4Teh84=_wmBd+m3id1@eD3MO1j-5C@uaqADjTigR*U%@ANgHeF>yV>ok--$! zi)&w0hak()acFtiNR}hf;JyyJFKiaxf~Ed;o#J=9f3m-IdD!Zh zZKyycs<0h9uoGja-M_lZ1o_|A53c{ejDKYJM_vCeXD^;@HOq+vC6g;@Hl{*WO{ApZNpAacpOP z{6O{caL_&r-_|}uHlZ0UXhj5rqy-ndHcV&E{xMh z`1i4oD?^WcPZMm^Vfqo|pUvvqE}z|27mhma z7^X3U>qTS{YHI?T{Qfu3vK< z`_a5q!RHzJ&O0*dBho#IL|VCXO1UHLQ^KdwH>=!nt-W*S{glSH*M$Qgtkbutt`_bH z@!zgz>~|Kij4${Cp8%8MC$_N-;^Nqg+&bm?M)eij7QVea6p^b?j1rV$;l<{_zHm>N z)7QU7IIfMhHp=&hu#TKl-)uix5Z2Q-py9o>p^WS}tW8<`MtTp~eWW0i(>Eio0n#_9 z{I^dpo7Fj6=N;?`-35C>4|eoq*m+Y=@*i78}|MGga- z_k_W1d%{rFo{*~C6Nbz7g!I}y%GW(%w0KYGOx~gY`Hs+yo>O=5sqYBM;yXg$synpb z-VrLDuL|3-1C2kmKFL@JJL$VHW}oqrJHo`8JIw#P<9zvkRvq!Q@?Cj9P<=-j#1K+j z?g+!1y`yb+gpu+)!f2T|*R@+Z)u=%&Mw(q?-JVcKpG>FHzo zeHX27gE2n!I3_Sz$A0Z(zjnJnGUr=ONV5w`)c*tfLZ&f}3_8^{y3p-e^qla2LC+>h z_Mv|!gv0iE^7!H-!bfoo**9x9-piK#cuANMK8{(;;RLegV9vjPPdFLl#_GZ;auKI- z24_)l#P^RCD8j<$_k>ksF-j2U|CW+#uomlZVgB!W`lNg|!QN-DRmKZnod0{8uaA8G zJ-qb)VmM7+=bn!MqFDZy@MY}pOnYr+F-Tx zI^;^^d)qVQ)EfN~;u>%ehtPy(v|!;2cZ61Q?pJ$48#({eJ3K_?RE_$EZy_PB3rVz} zk#4vuod;c;}dD3dz16Vl_@-|lhwzDd6C;#-n^wqLaW>`M=N*Rc&1Y(ypdw~gI& zZ0t8XOb()bt{^~I zFJ=G9(G}*twY@u}_|p0IH|qSsNZ&^O5l2;zz;*JL>e`RfkUd#J$N~{uy^$ zXsW#}e4_OBuy4)o@X-_R2{Y0@j>c`bg#*nsVSmf6(11G=HQ{69HDT7aIowtK-f(yQ zN5aQ9R)<5l$GPt9stYGzG!~i;EK>;oyvs&u6?#epH@>b->f)v>F?{TUJ?3BpC0;7uHvJe?*~ko z2RF3%9R88EBJD-f@?4>G(sR$!7E=;d(2KAN#mL*a6~3RdOGAlpdvReXB^&>q|ER9I zmfo>eowuwotfTkr(uT65Fs!FcUVaZX@uhtB&)C;JJ6+lyR%*!P11QmtxGeOn zRW2xx;#`CL_(J7Fo%7VA0SEEq@ePNB$2RD{#DscA{FmYs-yoA#Z^`fv$bnt_e_R~@ zv{Bs%BdgRw(D)h8{S8H-Sz0Yv`0Vn~O14G)E7}FLZOnK45I5m!rSj8k9&@Ual?G|0h;>zUVG>oyD$ejq5@m z`mxzDTd)<|P=QKRVLQgu;l^j!82a^e2i0eM7EJCj_14;h?>_h*o@hn?B_if4(&%fRG zRvVAsyfk#6j-FrtC$Pio+2OVFSGoK}_MyMnngdmQfCk&^ZO3otzj|pj;2;j63C(Ch zE5>&6?{}*skdt-i=lo@av%5yYRv4_@e#%XoPt-cL1){*o6ZzUfQajo0lH2a@n|H*#gIp-{K zd@0soE!JT@HehU*{&igF=btKJmys=ErwjQ1Gwi=Q!4R3U-*Bz|dBpX3MtA6cx2;S% z8&QtU=qOjtVGBLZiHqakx6-$vKcoE}N%R&hmkE4->cjhXsy|ma@4tD!)zU|W{Tsh4 z-)oP`uMeiUkm|7PtUt_#=ls^vdSU=kPC zf1UA+<;DE{54wM(4ype*Pr5?;A4X9ljdu4~OV*)7U8>qMucs&I-Nmk}nlCSW5czsp zlR8}V;kPdjhr~6Z87*i<8xrV35`#!%98b1i8R7kVeb0OORF`YJ>-DdB?a@cX&wuKY zaFjfTX~el`Gvsm1Vh$&85~r|;lstEuyfD814E-z$j%q)PKKcpy^DH0zl>Dh~o{xMP_e`9Q&@&OYYogY*5 z{Q5+;#j!m3exFs!t~KY^EliMWop&8N)-4I^$ujh`X`j&cm+a#Ek87W3);@v#`Vg|O zEe!|$O8@_l&EwUEwbA~u94IFj9$FeUlUuMA+fac@RAD=IU?+BAH>y#Ch2dC#FAZ}a z@qGx#e*o2y^=QCBq}0a`kxghu3*y+$^l4+w=#y=}&ExFBe>_KA154h5~r|;(>Q~(DEPAcf871MrU}zT3gs=6!j33L=bL+kP4ZZ=avEN$6u@_ms{yO?Xr?C&@2IQ{a zz{kHPl+ov2wmNJi%dr_-uoc_Te#|&DGNEo+Nmijl+;(yY;=gs`zkrhJ#l7g4pW;7u zy3|2CjX~%suziK?WN(S>>ePMo{qHUeI~~6ZyHSmOt^8)}#kewM4Ec4!$PA{CLl%Q; z)WNrUMr+l zG5m4w_l4E~`Kj+gc>X!QRdS9`kc?q{fGN*vkeU+e!b=yTpBt=*_boEK9=)}jvc-!(RrjO||yF|0r8U<~_a50P>FTNBxg`;3EY zAuo)7Yo%Wp|JFuN;J3%Wb%{%25bbB>Pcm`J{6D_xI6Z^I=vEfQGNKo;jEH4JKb}0k z;E4T>;ux}TzC6s3ji2XVUu#SceHL>#fs;6e*!R#SpLe4Nz4CriIgnp}d1JJ{MPbo? zr*Q^nQScStFIHe|z5Fk~UF`pruk!1^c$aaUM}Ga~3i)5&Ppy;xrI}tU|6{b6Z*R6NHLN0I(xY^O(`H2#}@2Yn~{ckoT?_^9YD zSQ&PSi$45rvKk}w8nPB?`h+?7lc*D}N0y$$g>?cuxAT8@JI7A`Z%mU+3ipxy<5z?R z#~j2VG@%(SXhj>w%rPG~2VtVpd3H(H*n^C@2w7tfayyJaIBMQHhK_4%z%V_H5&Ome z|0Sdm=dq8Vi@sM~Fuy*(exX#c^h%{Coh)*WN!n)+6ShsR@c)3M^n~-{f4<~fpm&H% zAVcrj=H76aegysWI0qq#I0vD#+Wn&&J?L%q{WbXh(4SUT|2TxBjyr~F%-}d?F^7xe zPj;|Bm_i0wzb4-wQfG?73F(}~DJ){7P(EZEPtzxNXg6Y) z&(O~zw@ZCjT&9Nqi-KdmA!G0h$rb3>Y&DZVh*PIPpN&@qVZ+?+CN$v*3s8v1IiHB{oY8SxhZpIdDMXE{LJ((U?|F2sb zD(ID{LR|A|5*OAX$tqK$58Sy${;zbM=-KJI>Ri{L@&$2xQ|$lW?)V+piCx%@YSbXV z{?l&xf2ZePajEg`WR3`;3eK|ETf&>pXul&cB-bpgj8$bN{XXAIJIEO0y32 zXuv@nLNi*>iiQ6$_utrnHhOz2zeV3rejH|nGT&SNYmt9V+n+{OOxQ2(IM zB3H@&BD0hI-L3v1uHE(%zAY5fOA!AZnBCwTYw)dMjcaPJSsqGlTZ4|%`iRMOh-(OR zPqS-SFT4SLzPq@tMgBjdhYLcPxQ!^sW^BP$Y(oVqQHAZ;ft}ce-N?Sz`n%zlF!u%h z)W5HDP(QW*gl0x`ozgiVXbXySTY+z5dzOzga(5=INnr&R7Tgwihf5kI)mX=EpTJ3!fXmG~ZA5nbR9XTcPy_ zPTpcIwDPdtz8g@6jTk)qJ7GZIS-zjF?sr0ZM!y)!ZQG1}Yu1L3u3Hs$<>i8QdCr zOK;`>+{*vCmH%^V*x{I+*oEDwMh$9FhcWw&7hC@TlWVPiP;UK$v!40sO~yZL3Ilc4 zKiF*j1EesF^fv1s*l)D#*3e#mOQ@Gl0}kR4IvQ>XiRN2E6TOGtJ$_4QrnjK~;4SJV zx9B^$CG=wdm$bW`Di5u;wPECBc}S35NYf{hJ53I5NK)Lk?*4lTy^#{oQs`97Q@6(PSM+Qe?T4Q{F`E^`8>ZA%GRQhd zJkNl%2Ft8JfmC667&V)SB}v;lI|bV^qxic@7tQ8A4mUA zZI8#@|Frv`yCuwuORY1HpFDv$_UR;f3h^Ii6Em)B))+nE)5z8;7ch0ib&_YqokjZz z`S@RyZ)L{nkt@(aFCtf=XO(>H{*q~BxBH8A!#>RaRKDxIHO#$old%f7hGP4cpcHG6 z_m9+R<9wR6!V8_7!a8z&)StU4bgsAl?{0msJJ0vQ_8KprG+w@s?AIrI*tNyAld|7k z8a6nl@#)%%KV^&seIv%$=kaKx*_t?3f2xpeE;>JEEyuPE%;^EG?wqt}o zx`X}Q$o?8r-`(w)TvOV$@1jq#trMKZlT+SNyD$vVV;2iB7fNI14z-6PKHKPY?%eWmIjr`0posAm+r|8f35 zF8=?kT-m(VyIs#tlTD6!a{EfNa0^re5%TSa3IFGMnfCwd&*zJTw7U@h0T%!LHMz$7e>XcD&q>c| zLyv2FXV5;UejE42I5LR7d;Is;VS11J694yegnktLyX7Bo$r^oq=yJ`SO~xi23!%Hs z*rZuwl+dp|r%zn=g+<|* zsh^Sizu@_mYJa7dB3jIH_xmLGxX!=U*x|Zn@;@ey%LDTBe_Q`TnuE%c zAu=WHVS4;G%!u=io?#QDu@38zYBG11ys$oP8GR$lF`-Q^jv<*k%Wjcb#JOx8i%Y_0 z+qWQ&Ip0dQA6*i*krn96uvLfDH4|(Xy*sU*O;*}gg^Tk&z5DILS@nti&srLG(syAu zs!@Yl)S(`Yag4pbwFde@96}SC(Sn5m;D7S^Y+5E+KQov&39XOS&h>`9CrihCGEuoW>cPMZv%FH?aamScQfExFU4y z)V4w|L1MRg+GKmuxE^vX`sUZOqPvToLhtzbcIiHPKQ`Fs z$@ZsAcq7V@eZTrwlKoG!k=hL=$tmHCw)QM?7--}FV+bh>BhHH(NwELUyTiT3Hh{Q( z>@4%YEX+h)MLy(zMasS8MKRG<>{>c&HQ8IN(-@bGX;t-lJQfQx5%R@6gKmT!~dFk|4 z{Pz6EHgWCVQ-bV5$7a`0?)Tj%D&1F|``Y4qltXFpecRoaa4(s(y|ab?d)Rk@9`qt9 z|Mtn({ixShzZc_<-FHSE;S=WHd+!Z%iczw-Mfg;3HKTWo1b zDQ(eWi!HX%LYubSDKj$4QD)32Dx(}_lo=UiY--V_{D1^VAwYl>0t84y(u6#d=Vv)1 zqc}umEJqoev6(bu8H;H~Gs@Vs_r257({tvWdta}6|MsUm@n2v+5v-%G5#q5YaySLx}p#$GIE& zQ3y%TOLIuY$oNxlkRE2RPe<=H>0u7J`P;&)COyoh z&%=CVU;#4Gb;`U&^AbDENo+SCv9I5}Mf9A0bB%vfy10WzG@y;!R`v51G$Y!R>o}sd z{gtTC5qD9G8Z7dR#aM!+ScWVt#|l(lw7nI(?|$Ror;LBs8vpJv{*6|8n{oao zdJca!APViF(sOO!N{_}i)f$t}qN_;=r!rV%>sf#sTWadu?dTJ);;y@8^vJkmE=>CtaXDU;0OS)3rY?n8)n;<}WLM)fLm}6@%7Au%At@UhUgfn%7L9i~8lhja@Bj*2cp;cJq;e z1<1tUgR{aSa_A9zIgm@R6w!G6GBOLxu>vcxagJ}8rmi3lSj9Z_Qar3Cqk6;|azq_I z_lxRG>IW0+?;lMG+3eRL2OE%wd=#J%MHp#Nr$1(n;0rg0V&*SrdyiRbU{1+CN*9j*5cX1u5NW(Nt#|)(7?emX{l>g=GB|TBS zqi`{apz)Dn>DF4+%kN3Y-{?lve6Z`Z? zYpb|fjWt+{-Up|KY;qlP(0*F_N9TFzQ+e8bN%}-@vGj?C#qXG(w?g^P+_F^qCfm4c zTC9%2pFHHF0If@v|0tw)<@0+(JQUH3(Z}4&+(UL3#zP6avLthO$#PVr`W~`2GSiQr zlCSl92j&*kG1oKK$bV|diT%B!@>}}N4f+zf89XNKkVD2^Pm!n5kCA&qI7^QF5?Bd?cf@eJ_#YNcxF5g;blb*RTl?>papnOKCySc0WkhV~2Q_oGu=xGT@R zcTR+?QJC5T&oqy<^^W9T`LKGp&<0txmddziUZeQU0(RteUBU+QtgXoOEJ~Yhp z{TaUB`%avXFx~gh@cpU2A7xecKj6=s^KkudeFPP?zCVI}^{98dVEw`!#uskTFE-%& z&-(sh-_O2^o2HA(0J0Gib3e%rMEiVBj7zWaznM>>m)^I+zmM9W^h$I1tIX>^K3n=n z4QdhX@mY^k?m3NqoW--#^qJ$R_b%10ZO@lxFwsv~q)n@B-;*tk=1Bi!!?6Cri^jjn zrg3xc=q&^K$=ut72u z9D6tli$fQtgp%p@8(jQu<1JId{^jou#mn9uo>}{@uy4b)VQ<#ELea%{h23XwHqLc( z*wu1lC{zc`t(q0`=FAG4v8D0aaM3eHv2poLVbijk!uG}Q3I*lYhaK42alK6V_eo=} zKfT~uXTkH=o{>f3(KD;B4*LeK4wqaTv7i6$(^rN)Iah{5^0}cWtY2Y2xZi$(KlQJT zA0Lo+9h4sqCM%D(ri7sv{xzQBS}M{oBJUa5Zcf10@S1)Hc~9iehS};F3tVfy7~d!_ zh}JJm=YFF8b5}~3LBG6yA)OxWNzsYf^f`Dt{i~ZsbJa_|*EIHXF%R>Rfd$A!r8Y*@ zuhfhII7<601{;cq)>|6g)qe}HxB2bkD@Z;rBenld;| z`M)G3Eb^SiSc0XfSZe(lmeH&8lz+&gFGszyuny7QWi?pAZY5S>HP&D)vat?PSZzR* zU*<4JV@A2iqvxaYl=AH0>e>o-X=_RmcLHd}kWWv2dkdJ&3Ig167# z$yfhJcd>MZUV0xIq_akO?Fq8!g8n!8a+Ei>(%baQG_A150)NU;h1_n}O1R1P9?V@G z{$C?M=q>2W<$cW@<>EYbWHLH?^PGFmV*rDAXZ>%OxxGmFU!sgIRsN%Ut-2+8(T9c$ z%3hopQ`Recn+KHtEz18!4rGb}>g^^JoUMe|sm-!LoG)(L# zJoB}9*HX{Mbape)H;td`)DPBrmql+rGxz1!;_0r<#vHUN_ggTRek|#=c=?<+;uYuL zh(DL~yZAhIrPJ+^nyzh;DUX@<+jwo#tMR&dzmC@@{W@Mb?f3Dj;@`)QC;cW~o&MW+ zbJEN4mQ}xu_kQQKcrF@}UWqqmjl@qZ9f>!s{Z)KE|1z)unOKCySc0XP*#GywWc`Zz z_g}suEMu32IGuX?%sbD3{jY5q^a>+!0h*W<@aUXNFkHI1*wYkOXg*Hyh9ug6OF#ncqiGF`9{2(>|yRD`}*IAH}InoCpNquZ$dL#@?Vd)u6`rlmi|V( zlplk)_y^?B6R*Xq$XYa_1s&+YNu0t&`72&+#K_d~cvIHv@zd=4(ff${8QGfS8S7q; zpQCrpdp+KXiTQao;$+(E@!piz<2}<~k9VW2=#BV!?glW3VN~?I5ijp}BYuHieesR> zaa^R2qTV;voq8i)d+v>R4aV3tjlLegM2@3XcqaX$`W;&6or~oQOT}O2G)$Ze$K0JI z4U?7LttvzKXA1u%!hezQC+pSG60uGU?UImuT+oVtTLf z@p|TH?rkj@?cw>({coc6e@nP4Co8a&KC%AqI&JW|>fv+bFY=hT`xm6MW!z+8IaXjL zR$(>PpfX4PBwsm>YSf^%O8!(Re?r3s`AdQPB~MzHUp41T+dc9Zc}!c5{%P;gA%Ey3 zyH>~($R6fivhR}lH1e9JG-=4QvJsV?>&P5LYXCc?kuK?H19KjtwSRpp)N``cbLjc( z3Q&k56eDUsl#tO{pi;6NRTxq3Ow8APtNvcgZld02F7=ih^bOpV99r0QpeJ%)sohQv zKK6S26gl*&xK8%tEY9IP1~7=G!lW=vUS3~zfqoIA==86<#E0%R;>WoBc}!Zkp#48r z`In`fz(oIdk#eU*8B0d{gf_Z&43{vDq@Rh~NJSbZ=KrN=TPw@E$Zl=^9&%#*JyZLC zj33I>6J*mWca;(5gw6Bu>gy}>*TNrmUNA>E7eg2ij4FkTPKPULtL{5x< zRH+{ozd8O9?TNICKdZ3@YtcGK9csS01@vg_qm!F;^c?i9(U-sK&9RT}b>+ zWIigmFCe2mh$rSBxE76p^oo;3?CWOxF8{nH(|2b0_EXA$Ms zCD!0!qqG_I1@yjP?`^Ub9sHj-w^4suv*%# z-z(bm_5oVPJamISNXX?_ft6?)wiblU&AmFTCfDHd`oC;?wD;u1`oE)7!#d`^+2ZB` zadf`8N>*Of{?mp!uAN#<)V=4fCJgf)4bcf?fHTz6*M^rr@~t_$m5nL~H8n7QETVQA56c z{@ya*kGJGw|h|gAHC>9!!+-Q6KF#7sP9KB z+R)_PhWRrw|F_+I6nbLa;G?>fhCjMB%@m%(rJoboJOVpmq8eMu$_`1Th2{*kFD z7uHeVNi>Hcjd>dC*1q{&QQt{4hvDt^zq)R9fwW&N4?tbMJOI)8bJN{71JVB5>Evw8 z!CbuE{&!DTp>nZ9dPGzQ??b~S?SGt*uQbV5ng_K1$<~wF|J*gTSYykd`N+Tmw9ep0 z-6E6T<-Izwh`t!nncBUs_qZONwY`L0vDD9@6ot@B;j{9$FRZ;AF4 ztCzRbkhQM=zn%ZJl0U1k8fy@p1(+z-4wMIM9E;{2$^Y^8`kw)5OWo@2{U3c_V~_9O z;9K$b`HSjp(f*HX`I(J%7&xy?1|Gls1|9{2#70g33^+%CWd#E5X+k-LM zf4GodgkqE+TG!Z;txuGGdH>;ZdKLQ4d$+URk(+KrXTC0DUyDYxpu$|ea`XQ>=o9<@ zj~b7ppG3W~D4IiG%S{au%jAE{<$o*nMW{2J;^y-D`_uG^^?#p`e_vETQ1*5s+PA0o ztoDDueSNq&i*q=S0Ssc~k>oHO>G$a4Cx`YXhl}JW#&8Mah}ynM|0F*~D$-CfZvS3P zqgNYuIIgZSojwEg#sTV9DF4~jxK`;Ms=Ui_MC%0pr}KA9%&T|LY|O!2yfgjJV{Xsa zzk!MIe|3oN4rMcXmudf_VOaTpQTacv{8tB>=pQ+y{6DGu=dRMfsPa#ar)sC8W`=e- z>ge^z{k;G6q`rqr^&MqUhG#B7CKg4{=cWUl?&+HLW}e=&z&+}=QT?_-{@=Jr{eN-P z*P#ASwk%cuC)>DN%#S5lis;uq4bI1?6OQgL#X^`7KGFs!i()Cp+ zm#$ZnW#{ZkORhy7y?)@$Iz=sc{G#-D)|{E&*jL2$%K640aU9jCDKrNVb*MLQZyh%| z*nm9bqX2~{LVJ(-Ppiy-QVw^K-OAw}vX{A!Y*5#=ZskBRa(Q~EqBYW#Tha>$* zN#O!HG5_K>GedO#-$mv!dD|#ChD#Vng*a7y-hNU)7ynO+TbJyQOi#tc{(ru?7B$?I zav$wWvcVqr38Zl|4bw3Lm8rArPh$`K^jV>Lx;^jH?0s*1zYg_im^RD&vRUDT{?n$@ zv*hoyLd(fn;`c25f3rgSsac_eedjr0QDuHWhxq~MrT3M(#~S@~&zX%mn2Xl4{3y2u zo!%AQJ7)iVdItJDr*{EAGWo^b0(O~LgvD5b!OzbMOUa?;nPC~3h20t%A605Kp zWf?bzHRM`kV;w4ttC!C)FM(cdoc(yVe&R}PK<4^#a|1ezo%a|+=Pt2S*ky@7XgaO` zo|_io+Dgx-Z$7G?lq?P?6S`{s^A`VJ*!PmrUJRAR(m#%;N&hMSNxFYS)}cN_KF^Or z6rmU;C`CD{&^|}{pU3|N{73g({-bw5{68oDFO~keIl)a6*_6neCmB#kTxtNFf zxV-)+gC4c{JF$SCiM~O3gFNB#_;*8x@);*mg%6r%2%i+;gEnE+PEMS!wnq4_622>i zFD|dI%=_%5u!z5lu>|v_r={dFWTA4t{)+|jz0<-EHRpvN>ge@|_Usu<+#Hs3JJfx1 zi1t`m5$WPh?={I`6@4{Eri!QJT4ZBgWcS$3=D%707nQ+_f{3DzoBieWKO5-PD?A$$ z=l?vY&wfzc8u#B(gUUQ@zyj@GvbtFLr!0&1#jWenPHZ$kj~{u+M*#{^gkqGST^ZR? ztNbfB{++6PpP_um#Qr}i+W%*je>j0A{IBN&MtcI5@~0eCs6}h4_HUB==v~wKjTU+b z`qn91*D7OEl(pz#H+VEToFs=nt3Hw@EHcEmOk=m~PNN^WpVwymiZt=6G`dSaA-j?N z_Wv)K6mlmkTZ4Anc5@ooSLCZdY?u_z)2oZr4KP3-M7_R)I(-MVdFl@sX4iD#ig1Cv zh*tV2IffSc<^I1*%;We^{eRlyRhU?(m?`|{2>*G)e~Iv4q;4}PEg%(Xn1<iSz$X{Xg{oaW|VEb1)ZW=9bMP%gGANr|+Eoukj4>c$)Z| zBL1d}zhoVAJ-N@A+B2(+tu3Aw_LjUm>{1! zsQy1UEfljaK`F{nvBEln<@yEa)k};G=R|wHiO;Anw)a~hGg-4N?SHQS&oh@ga!0nF z)(=3RsQ*uv2Qzn_(m!yLJ$liHMcg#vBu?Qp`f(QLP`TI`_Xg_^++U3v)XuT~z%%N} zhLzSItg-$;Uqcg`SDW{b)^zI+rdxjyy;Fw%33R1+KKq_gc`N!*w(_cQ-g9!llN1KX zK~&_4C+vsm)%n`c%opevQD30{0JW&WD7!IS!Z?!tS$ah((l8D0oIgFCc?Qza+bsXB zRsM_b$MHXv|7ps9abj~q-Cw`Y9Pa009_Awh3(%hTj`cq!>c7lA1-_5$>v^mGugp6! ztbU^0Yrd%dJD~onZqVj>)PBt5&mt_w613)tKSkQq^k~jqwDYa=4-7t`4z67t)w72l)XwIAK6L`%KAU;{iYn_5A>Eb#vjPGC{ARU zXUCrmEI=k&m#c3qGk2QaRjLeMY8-&R1btQNAMAR_?iTz0uv>WwS?r5{TSAgW&Fq{C;I;$u}?I;;=J*Tv-WzUR}16g{rdUod5HS% zqdxpvGU}_yXV=uDoj?|%RheEy7NdpUnWbK^!gv&ODK7U5M0?Oj=U`N>RX@dXRO3IL ze^X;Etoy1^i$=7d13hS8ZF~)#sru*9oi1-kkuRj_|05f;|KC3UhP$`-=cBjD7f$ly z6i%Z|+oGQ=pRfIav-EQ~kLYZG=!}3G^^R!Y?#ui0j@+RBJ6#w{>rr3)0Jp=~{JRI` zo8}hKFJcs982ZYjFk74){Ia#T%>Qrwf8*RFy(CXZD$+0w(=h|>+0t;1a3;I@rD^$i z&ww<3TK(&!{-2irL;s)u(zaCIV{KnLf1qUK_$vdNju+i(DRES8nuv!rlwJlEMn+A$tg}Bv)ZIHvd%k{KlI3lFTsZ8<(+L zgSDtQXAO5tX2_;jU$QoPG&8KD=b(OudeyXtLM=Bn*ubu-^r4VP=A*Utp-?~;qNUMV z|0UL?qlmc}z4ShGqX(5)4~42#51FIzP^czra`?U4Gdiq)M~Qn%QI0Cqq7g0VKzo+; z?<=f-C%e}0H=DmX_G}>gm>W)8|Be&qt$#-|y#=k@w5@%3BJPw8KFm{P3MwwTpTE7@ zo8_a?x_s}2+?R~Svah4pqXxBT8h9|AVm~pzz>lGD$j zZ@G6{<{8{{qjH(=$He!qEvvb*NQvbRxh=uCi7g>@hdumrY1`|&W!mOX%|uc*!%5x zi01sv<8F&Scl4e^$^}Vio?s?*9|* zKe*ccYp@pCSchosUk>^Id;gzD9u7n55<`=%U%fYV*H6|DsBt zjI>#cI(j{#`THxmTZPqFgSE)UI<&8m{pZOWhxJ#jzP*t@Zus1H2an?8rXz0zN+GB}T(k1l0zC!(>v?nZt8Rq_F{z_p6i z#@qEnaU}ghJp-vo!^rdYfFh&z|8#Qbn7t9E zP7gEa>6ndZ>~jt|7xOS5m(TyopjWr}HY}iL;_|-5wdKAQZ_j^UqMfl^yJEF=1?q~m zGcfAC7P)URmS8ECAq&gVt_{*LuAM=4X{Sug|JPpb#l-yo3)&?(!A;YscF8&Ik^$`$ zvaO%pNqhK5^|u!J7gq2yS{u-!jnk`5)-{jc$o*nGtaPn!2K#BozvpHon+c1Td8^gr@A#1e@%mcW@JdWP^+6CxdCJmyY&>W#+bAw9d zL1;cJ?V=UY8C~sT+839k;eP&{a^Haa$X@0?GU;V$2B}Cx-jg%JG;%sNh`W`urElpj z+5_wD{>xGA2h?K*cj=goIT-o4{ieuyn2({{5HiS#`Ug2Ne@|YpQC&Nexy-vSCMV|K zTi>^YUa?Gm*kb-8y;}X}c((o%dKT(4CA!`>r(xPL>r~`@h}z#2wPF`d}IXU#WY*Hmn;da@r;z2Yo+4io1GY3oP(5}s!sz{I(Pi`7%u_vix{ zWH*cpxQJ1V;SxqFQ^Ghocwb6Lx*+_W>6k*MA`R0}mhSw$d2iMsqVSo{uHu~cSLc{P zuO9XO7qs!{xpyejPYTxo??34M+5d0XANUt_{H-T@U(xsH-p$4+w%j6a(&u45GSHqQ zoczPiH2;9?7REiP?pt900W_!&HD(KY?wZKv<^I89VekL8k(GtgN3l3sDt(~lthhNK zj-GpSd}@JbWMUEe?y?@1T!N)ohIZ+_V}RuT`qZZHK`;6+F}_xy{L54R>F;aK z_K$O80nOUS1NV{aP}bzg(#djP@Wu-sAhpnp)p4UDQb* z(fDicT{ENhNg@yVC_oXmeA)bfndzZjn(8P}|027jt#10;Bhe?Kp#YX z%gvzp4b1YkGl)qh5CQdi#{~)qmiE{(1hkxa|>|! ze7y<(opUqtiS!^S!Z)I4bbIEHyyINs19O~rR%jmg^z?8v&3IMf=I||UhPvtB5vR{t ztFL^HzG?ki_qN(MAll=<=0$VjgjHubv+v#HdS8L{`{-fUjk4*+d@nz5S_nH=&j}To z!hgZdVGH-w*}{JnH&Hm~tFB)v{87s;+NmcyE_!FnHzRt-=%5cnbB#XDDN(7*XEfEpbvbnKD5#IgiQtS3%Nz_ z3weWgh0W>j4_nr~Kh${cHt+Ft*4<&-+;=(m%zC&P*M=RV*M*(w@1FRsov!cBx<2eF zydmsedt)fNbYpnt(#)_g$Nmg2#>1g2jeli%SO4jdcv>`7-3RUYQ$kHrvbu!)Zo?Jf zjD14SXfu4zwLScNksQLmAeA2*%0Cc0OP?)19VjxMmUUaKyHI}D82xYgoie3ep4#DB zC-XjjG|O*W`sH_MlkZOK$1*HGEYbH%=6&M3@q^rLdCggiNmD`@eWqcp=g95DQ$xYA zq;Td{V~DC^y89FP*Ks#vO4v3vgi_|$@%P^8dCZ!k9R7AvFr8JqY)bey@^wt&-xQ>w zn!j7gccY#@cD@>Y_~Gc!W`UzJ8>ov3wx~}ct3sjb>i#itzitY z@FC`pp?!2p_!F{HIDLl9pL=EabMlC1bl;<_|Bie>S+bJd8S6OOPfZD(kE$Q0hQt@y zegBG(_&NR!Ut|6c^6TW3Ny;1CfJ?X=_xR?8ObD$VX~Efe@-rt6&jO|x-}EuC+{S0 z<0g~*9Qg<2O7g?5t-%uJIQggKlkAt1k9+sekzHxG#y-j4GuM4EHj6o$6V@x8MQ3Q* zvNMrbt$mkm{B>a>~_$^9v`@uo1$e`NMlK1pLX~Dv+7~W-2=+sgYw@)tDWywd|m7?H%D+Z z`?^?Z#}(n(CD!8P&5IqIt>1B;en|GuwJ4uUr^Tw4T^E};pXgTK(VY?Ne=sBF^%KW6 z-D^;fHrIM_4lm%e>sO<_R9f&a+o0%3e9tvi)5JiEf6A^iMILode7In~=eWIRPAgl7 zua9-7DQlCI+k^JFLW6dFqq(B3)_=Bf+dO6unv2$jW~w`$bdDZbm0{n=esfG*t08Mo zrG#hMm!+hHqn=T^z`TO=l<*w$F|vHR{cgO&LH9pIX5sKaN;on<**LlVyUwMAzhRe; zW8V3Fw}r&de8W-tZvSo+W7y-Lr}+P=I3oVjn*qP@e`N7zkuiX;s|IvG6XYQfj&VK5H zvHmYV96NKv!?FJRBAFTM|MmxBXC8f!z94qy{tv|ZE19>Fzq>bf=1I?aguH{COnz^6 ztpE1+#Lm>`yX(feK-3WZhq(aFWw*Pf9(F4 z;~*3NE__UhPh2gY_uuk9_ud&h&h9&!9ldzAb8-xK?OGV}cl#UK7%vmn<0`G;a> z1ZHohvA~OW#s0hNUGmgP!l)zj*4PjE;b`bYIs5nW`vmz_@kVvTdC%I@^!gs>*%iM( zR$D$b{5R(6Q@6(Au7Ah%+sGf0y2TQ6=_^o{bVKYYyBqnNjw36khQrIInjbPXyqEdl z^4YNyWHZ{(h4(?1UE;^sGw(gI|4x3@zy2HSOqm*Xtd&o%ni`&D-nRIb*cZHie#YIg zzaak->)C&ud>p#E68|9$Z1CK~N(AZOI--psIlRt3_ZsQ{I{S^iG&yW}Fc!Anl@jug zrG%%$6Cox%x9_?lTtQxiDQK6M6n^JhVfTl>8TNd1U3fRUGv=^8^J;2nmyzxIc51kp z-EQVHzmtbIv)AW%YvgXGbf;eM{z%vF-LC%e0s04#iBF(%`{Ypd(zn8A=+#dqhY5G? zU&VYzxxJcf-21K2t`5?!ALYzz>Mh=VO>|wIWGxvzqw{5FWacJ^?(k&j`OW&!`@5Xb z_uAh`Gz()J(iX-xCM}F@N?91oov|>Mm%cEzdD_C*mgx&)Tjwr}<EMdawE3%eNOq@8TQJ z2-m{M&0WfSeihM+BQyP&`5NU}_O)Aabtqw1S+y`$Rl6`2)$>Z3+cUI5Bex&desA$E zcg4e*>zs4RuF3q4X7fAF1nrAd^FI)sAKosrY|r-G4{JNjy*l)B+iUJfCKJ6hfw+c`WX>{>lp-Ey)sbti{CYvfnAr-Y*P$>Eti_s&dFpD|}5b8_T za=4ydea2+-3nz!h(d6)>xc^@;IW*6`Bi1r|iutWmLL1s=Ul}^)Ob)t)6J6>=-D&Bu zo&oV<{*|E*3Ex_!j6dF(6k_tFn~*mnHEb?U4O@y*!`8vnkUwWm?EUN);S=~0zKH_t zs+|%Fm!z2gl@j)>ObL5YM1N*gO4v6oCG5|LzCR@#AP=UcghSKKg;Vw&CXb~0{;Jea zIV~}Q^TfvQbOqylftik%Yd@=7`^Jn;pCv%nYat8l!^PG$(8sQoS14XCrk>zq`!(*m8&*q2aVDH z1D*7)L;78E^}DDPzQOK@v(n4XB!^`Fq+(M@4p)(R##72~@sH_SKbjnL^(9{QemkS` zW}f<&N@VS9V>ojsg@0z3u3Y&8H0aOyH2Dqez>nSc3R2vAGgK=QkK&W~OLY6TjRSYa zHVxkyON>hgm+p*B{CEF8cW{b!ka#b=w_z*tu_G@v8g_Tw8QZhyy|G=z zcg6}&T^XLS@5jFE_r~^~a;ERu+hYfq51xNtY(H7NZc^Cg-haoT@hgKN4*L(t<6xL4 z(EwAz67A@QrbOb9bLyL>uMd^eQp0zczlSsU5jOtxsqj4cQ@nr|@e*E!s(#{C{06_n zYtURsB;!Hl;2+&)FBf5SbnFwc((|8)J$vC3v9hu3aO`5Xy>A~66-AGS=XxGDKlt%b zb@9n?{N$6NdUlREKRL$DazfqooKT;Z6B@E}#J?QzFUR`Lr$Y0Jr$Wopr$Q^eZQWDS z@Kd29>nVB8Q=yCOcC9DlDRqvgLf^cng6edl#(VrDSx+{vS#NyA{48bJxOeO>{Zy=H z{u5!+?b=x%2#F6ppni9!w2bfKC-^yjiC^RQm~>Z2{8xMvPvZcNqZ9v#f5JcG-Sa{s zk+nW-ShYTE^xRF^>$QK^hrE*YVRQcauw}#gur+Ue$S+tQo+h_lTpzZNtq%p`>%)!< z>%&g$DqbH7vAb%$bHLY!y&daA5uTy%!+zI_mpv5@%y=ps#G$mOLdn=u;qdrV;Rtz@ zxpewd_M4Cv$Yadq3!VxUbDs*&q0&87tDXwS*F0r@#8c)+Jf-dbRH$RFCmYE(|*2mPC2J9h;o-wZn%?c0<7&9JNR z&tiph9}l}XJRbJ+KOXj;e>@ajcw9PoJnTEWJhs2|@ld?#)3F1ikB5UxKOH-C_VG}% z@|)rChChoPIrq5sPfjQ;_;l>qj8DeOQXdb;W6O{OMR_$ERae*^isU z{*CaE|Mg$+XZSS!0{UJPn{gP=;y60-XYZHBaU8w)JA5Dih;hulTmLDFS5FDLHWDkb z3ahaJd1%DO*S;Axz5LD4LqCPhU;9?rf`0k{Ucqm0%RTZ)+=Y8^KOVvEN1hBHC%=M1 z^|+VF-{CbpIbZxk_47}L%9o!ERWCjn_A>9sQB>mDh9^S<*^Ph0nMany`ooggnb-bU zyW=CVm^ATj?7}W;pb?6^vTfjsV8-cI^QjCQrM^ueG_u? z)qk4>6apKkWJu@~F0TX^TF#o@g(Mq;QP6{G9ghKvH;)th{K9Vodow zZhkCT!(7Y0&h>h-f&0c5{j2V8BAc08vh=U6cyoMvVthPX|7?!FPjX`a1J`?9?<4o! zD?IQ^jNsS!E#5%J33+u53+pq`w(1kwy3_r&Zeu3X(R)%&E-iwc58+M`| zefTbZfh#|#9|xbrU*Q{Ak1g1Sop=t__%42kpWtWs1xD~|{1$H@{vmBGwBeT+!EbT6 zcxt%qetjyq7l~Ty?{x@l)G*pa<}&Ac-Tg6XD(pB!}Xoy zF76BU#qY)*>~(faQPx%dzq&EG-#BY=_ErA>RsO$yx|mDYA9no+d9>C(VlC$8_ZW*L zk1>~@yUPE+%Kta-^`i07G2^4-#&O9S=34f3uGfPWEJyqvYJ1~3f1ROf#+}(rFa%)IJ8WC z5QlLD`%#PoIEcN|)F<%_qWa{n#nRhS>5bgOynT-JI8S;ccV3{+prV+a1cjOhUd_RAL7UOIfn2H z{1U&$Z}EFfQeU_l*W)JKhWFx5+>3<#Vnd7mKl#TdGM72;oO$;H=G~K9FKX+J>Hix~ z691FLe{*fw?{Ix5xr_V4mZY%T{d>s0%!bUv1N?sAc4g4KkoardHb+_Vg#9$M#T(gg zG``TgUuDevq7C*^*!a>FVbj*+ko){iA#d-E_BPRvuN}DcJN7{i*Mz4l?Kf95Eo={W zgo5Tf!;bd%g`M4VolWsr7&_|gt#*EEf41GzCNntllnUyY1d{!Oj5&B&J{42_g&l?Ld z26K3UYd3H^Ss#aREMDfHlZTg!15vnsO8*AA@e{Fd#D9547?f0Uw^kX*d|01zaWgab z8~1DL?qw(59DBk&U#52`6Y0g;r1iP6@Lum3`8n}^-swT+;sRykuD9NIRGHVnJ=yzl zeYWhQH5|ox`YN>XI_RCoU>|i~*Kd+1zU%NQ*XR@9t7C4XZ!2D9-1uvrBhDONYtQ)- z<|sVM-@0D)mbsC6re|O0p403;>K+_0)^hl~er;^*c0GEhG5QPomAAUi?O|gsN6xZq zcTW^1^XGY{|?=^C}`1>OF^utNqrSc_6{&M_y&0^45E&T@k*_ZsTL_ui}pUuW%cM4?PN>k?-)|eH*Xw-u&wQifiSreS&?@qzOIxk3&^U_v?F44#w-up%h1o=NiNMtTg!>eeaRI@Mvn6UhXi5>q19Ipk z@%k)(G#-nGxpzHE6C>aDZSs=n8^0EX=Tu|X*WeEBwbO<)SchhEGw)3|(X%%S%fp$j zd9PyqTN~Zi8-4$4IQpiahsS-#=$meKME^v1JQKN#o;!oRu-o`4_sh37CcEb)=3Ov%M|Lm9 zLh@Ycod1V&Jo5$bpB}=y*hSB;_YG0}YYa0(gLqs#*BYmf&J2f_dhV(azAE1v66Rkg z-!2bc6|WwgIT0WF`e%8cS>YS+xE`gUD6V{qT@>yU@h8JO^Cv2|wnT2f77M?ShaMLG z(Z7hoe`JO-7efu{;rg4Dz4V*NS@=EuHuAl=6C>@ha5s5xq$j+;uzQ1>rt{vP-XhGJ zPwLmaqz{1XSgAjBwy}MBr)xdjbaUG$OnT|@8}$kB>kr6DxDr?6lNhUq${u@{S^>4D{>c8T^T4UkM>{*S&II=Vzj<%Rv&||+zvdnz8 zV<+uhj|%#8lW~_$5Yg zXzYWbWc-8S*YsDvt-K>2z`p%I2@glQfAetkfAB}>v_Zy4$dBRh69eH-BYW;XNqz<& zp??g2iqGJ4_yWF!zrt7Xb$kQg!g_2%;qCV9Ah%&BKF6;wU=KaYllGAZ@mu^JZ{Ssf zTZ1qBLoCkS&?o*bHi^6vSL0fg=}Y>wzJlk-K2+o2ec}!|a_q&}L2~H2EwRB}zm5%V zFNqE9`Yl<_+!`ASe;OZ3JR7^#bB6gb()^M5NOx0gNj%xzSx$O=fhQW zNiT6DWCMx!;4XXsvZ2I7cocsOSw!N`aG!q7g&0gc;aNAwhOYa|*x)tS#s-gG9h3E& zw|G@-_ye(#`yPl5J$P$ORV*=b$E?`M)OW@HgnpCvn}tu&{~Wi`-;2lT z8?hBTAd5*Hz?1kQ*5I?=<4$hxhU_pQn@HSC{}8(A-^70WSM0`L;~1XDzu^@mmRZ|| zjo5_TwdQ2x7=O#YB5YZ6McA5Ujhp(()3w?&rTQOGfE|!UCf@Ho{u{a5*`9lx^SM{~ zU1U)BnQ`l{$^Fd5=O%>%^RCkmJ1HDuE>R{QE>PCby)G2b(MN%UIDiry!u7u4Fj;u= zim+?+im>P06=C;)GkG%A@v+afXOOQxvlZL$GzzdCJFx>B5Oq**A~zxrx!8iu=qQ&) z(S=T2o|oQ7??odT(1a6+&c_p`Ru>N4dW($8JQpbr})VJIq|{f zZSlbe_r-^vd^SF`pMKAxF!=GB_|VMt+C+aCe#Px*X7pMdUNg`^fz{SBGNq0P{iq99nK%7l)Tz9gg5=$JL<} z&z`w+P93~)5S6I zqgwntj#_cG2K80eIW(q)(rn{&E3XV?7p@G?E)_QiuMEd7iJJ?eISP|Qhl+B0VVb?I zjH{D99md(+cek+ZB^%GD8KX`!|2-|7AX{8-X5Yr#N`8>LKg4o0pc6ZB1V6xr;WQaQ zn)1UQCCEcAwhUT-H17s!D=qBj=RxuSN|vOBL*$XQY1$2Gp>RW5*yZ{ja(BKlNpf#- znllm7!ZXbOByQa7Iot4kynrjj(L3=7e#QJdOrc+oyYW%{86Lxj@kM+EPvL3oMk!vv z0W_c-UHAvQh&OPxcFqhe#V!m-x_JH>d=6j0ueiI5{e$>J%*HaT#FO}I?8Ue7T^z$% z{0t}XZs}+;F3@iyUnTzo^XU)a!}t??7GFaSa&ZV{XuxUw7{ho2Q@z*qn2ir$0hZx! z@0E0uruUmHG1&SQTM?!)b_4Jw0vXw2xJlsVrf|5!QoKJG?dx;CVeU%1Nu z;MUrL#C+}z`zK!GZyNJH%;RP%zQoT|(QF*!{i2)i15@m<>bY1+BC zkNzoeKk_`jhg+@}X4pn&IOI-$IOL^295$0%n73v;9P%?C4o{QY zn76YpaD4~4lX+M6!=cb#0=vmQ%zN1v&3HIGllpMjhy4p44#hY?KZrxFmEbV_2##KQ zIF#a9dKr$nR*nk#bEsVKaHzs@dNpcXt3@5X9u4d4Z-5i@CN#U&f>wGP+85X#0iEgmMfu;O{3px$mH+3gc^SAOL~E#?W3C*t z_GMi8PgXP6u&;H!j;v>HXtBP@{S)iUms;PHrEheF`Tk@Zb9=TnR*pH)WEXQc`ySVO z$v);9|E{)Nc-9I}vXZ%KmGE98yvbw!b@^idW2yf^9%U{~@xT1*GP1iwc$N!K_xF+Q z%pG~cyFhr8%>(`i`&QT6$Oh)dQ~p=K|3z+JxiA!92X^Ype)%Msi%-&zl0U$=@f~^F3(BT%DT{spUC@cU@rU>VzKKHo zMERy$J(0@ot;oQK@frLzHefIQ1E#?xZoqr-K`h39!6)%~tVR(IM`rJM>($Cld_;Tj z#K31m)97bI%M@kEmHLv{{VAFk{7JY*)#;P;FOlu~5IeA5JM7<;h0l?nGsb`5o{aFf z+JU{=W~1-tH;!n?UL;FypBoNoBb4Cqn11E{r0~K$zD>LCOUBrb7*GEhuIATm+K1m# zzE0w26Fw5TaqVM~z2Ei=&)Dr5JGuFZ>jz!?d-lc3==IFom}l|l8t?YE{Qd>=RL`2i z{VdmSg!b^F@x#H;dCNbCuJB6e zzVBk_x$jr{$45fnV|uwft3p`y`7d{P-(w$Ff42`n#ixwlhsmMx^X6BE>%#FbPY>0v zygSr<{Q6M)*tAgB@UBoFmAl@l%sWl;{twWP@NdL>J+FbuCf36cgpCf-L?3eH>;r%}3Ic@k8_y2)=651FWuyNkq z%0KO&i`qZd1#ZR`6{xKy`Q*=yg{Rk8|70EEc6xy|6FaPr*h%inG1f=!X5K^YO*aNW zKEu3^-0z-Z@&NNe@{sE#X;UD$)& zC^>1)#%k>e^6+VMIfl*Yz(E{9-YIMIu?3s4VcJy^>n&l^bnE)9)h}ST9XqiD`NQT- zV;i1EOOH7_Xp^>D(b#iKXwZ+!LY5C^r}T*aC5ltT?T| zYxY&)D03P4tohHi>}xLR4tzZTbai8I%WM-}47 zZgDU!j{KE4^#|fnpXuyK|4zqWENnYTv! zpWITS{a>#AU#tCp?tS`Swf|9o9oUIogWCVtjXl_lB0PhA*#Cdi_U`dfRrmh)hb^>4 zpqBchEv-m%xLD9A2?-F9T2xf3RH;{LIhHE5sHmva(jMBGB$NB(e!owWNivhnOzz3# zDkd=rN+3qWK(WwDEmc}+%V}u~{@!ci>Ff18&-wRx&Aiul-S*mRuYFsW?^@f%_y-h2 z36w$^ltTqn?qK|b^sCrsHT$d~4{l^F?mA@scG~D}&eg`bYM|fp|GhrSzt=b6T>gK) zuZCaO6xR%NvuzLb2Dzr7ANv6PMgK`T`6%mn%I7Z1cI(VkU9_D zBrrNkNzi#|g*uARU&!J)fNZsM-y@rGHz5bgI9JdIX6*tq$GRJW&;@nNKC4=A(RF#v6}YfhLTJ6?h3JdVM_(M-K)$?4PjIQ~34e()%AkBkD4SNw{)9Qtl;2I1%^Q^6 z8=RLv!D8ao;r9&XIEQjy%{Ko){sI0@`G1A-|0nTcu+JghA8`MPxUHo7e>i79BHqnh zYw2*Duy=4@N17kPeQfs(JP&bj1$p=jvVeU&MjBU<#(H=J|Ht5Q7=+dM9fmce^*wkF zR&rj~z)xT$`??dp11XRX`^aA=`RRgJ!M2h6A7UX6;=ut4kO)a**DHAw_y5H8)c?8v zw{ic&yKEcJab!l&=Z4JQaD{$@e4Mh&3+Mh{#{GYi`~M90|2gh|WXUD&e`Fc%^3&IA z3FTC|=nCD6tXgq}o<|OXn>1_Tx7Z({eAZGH-dU^^94^sJ}l>) z-3DLcJS~UYpd8M^1z;MtEeyUuT3>;?;m1%2O;8DK@K?B(bMYwr0FvPIoR|NACn(d$ zAQisGu@$XYtU~6oaV=JT5#J_2 zdOfTifNTsd)(7VJxOQCb#j1pAr~)_CK;8C5szus-*UH{b-ulQRG9I^s<4NGS5|K$9 zQ}Wa`N+GYQou6~+j^$}^I+uA%r0 zWt?}dN>5Y9$=CA!qdftYm#&e^!TZn2`wv-D#`_OhyYL#-CG-9Z^8N!a_6BGy;`%2Y ze;(IA+cfR_ikhjvwvfM8>aTnjR_%m!kWMGEi#!EAT>qO{L!*!DKgikx3%*QymFvIx zT0KP>#X%js0f*sF@CjVa^>LNC{-Fmxge6?R55QWk{l{SvzQ*-_9oz-?z$5Syd6>UTKg9gXaOPh|i)&M&Uc|L2=G`zh3k6Vk9(^Ww<3rj_@OSt%eYKzE z9-IRo-+PsQ_7TSqi_Zb=#hc*Uv}YcIe}6xA0r}fa{`QhTbWalYNRJY}1-8QXXeWPg z75#ruMqZd=W~)e+K0^KmHhzKQrTv);dE_yFp;Q3%PTob~o8&Fx-2{!iSAF0Q^6r94 z_T}2TNmaY?8T%Yp6~X>=od%9vk;e3gCm^NxN6*)+!bx6m@h z{48k0-VPn4y^QefuoH5i3;fUul~4zNCGQvEGR(pK6`-5Z_6>-HwQvjE3HQJQ@B{cU z?1mTNXYfl%hIGh>66k?&_yRl%bamRUfXOe@#{j>EGcX0efp=i`S7;Z*huD7)e}wZe z_3{JOKahWeOYm2?f_OA*ZCAkp_!@i#u7xmI3AXL%|L&ZjrP$+kq5rdQhQ5eBVK4f> zyV3uBh%tWRq##p$=>N8(3)+YNFERsn=23J6xX!X62Yc=qx}b-+{)V~!4siWl;`+OY zZUO!!_?Hr|jCdhEwJGj@bO>BwjLm~tYosQIHq8dwGSSp+B5143qs=m#`tL&8EQ_fB z&R~px7Ht{)&G&&q-xuCPoe5g^Qis|_U1~RNshzZ`c2Gy!PF-mmb*2^6ox-U@k>H?iLVt6(F18=j{uk8xhEn~#nFd*c?F(qP&!-)~kaj$6;cCjXin4Xh=Ugn9E^|}%Zpx-+17&W? z-kbxd#~z^k&9_;6#BF4o)_s)ye(I~xguRnzm04Tetg+Ze`S+ZeUYjO(4m~z7-(>2) zgdW>@^w=&i51cZb4R=w_zoLv!!v&c7#{Jgk@&67y0FT2K;@nTzUhMIZ3^_+BE2JL| zfO+QSAE4|G;Xg*%o#)y+%{6$AYY{5WaP8qPi}*c-)Tm ztCY}pl@gIjge4P}f=oSomC|?^I3b<#$$-qwT>p^0jq4wBvFAblKCXW##9jo&gq1)k z_A)R!)D=)UM%hEv6xTo0V0T09Ma~7(WA}iUum)(v?gM`e=K`8GQ})oZjdKBQ*xSLZ zgWU=LkMCm;-b+{?^vBS)fHRcYm%m8c9G1drxDD=t``|?=0yhl9K{yPvuVx$g1~A-W z>xM7l?U53y6!{~;c`0}?J$|A!>( z$&f-=Dx_g|!vF8*pT%sv5sO4YY-md{10>fXR-F+6z6}&Jnlut)|)y1&_bR< z^S_Q#|2b)Nqd5QDIsep+g4~DQm#F_=p#HNT9jl$x|6ySJ6`IGhZ6(|UYvC5S4<3Y{ zK>}oh3;JOY-sk*ZfETgbnQM>=c~Ai!XoU~CPd|o#z%yK9FTl^>1e}6-Y|nB^wi{s` z+{L-Lg7iKIUxxxHg*NydX?y~|WVoq1Bp%!4nQ#XL~j>ew5g9z5W8e!9L4(&sSq-l^B%A6lOV zcL(-PXoGg>CSDJN2VK~Mg!khf*vmY4WFKyuW2WrRnLPhyDlTuP;)65gfCSG>B|=is zOeLS5sT4?sG;lhNZr@Dm|1+uo&s3IgCi8!0DyMxW^M7Ve>%XAeSFn}wuRV-^x&N2( zFUG7Y&R(fXbXr^ytbeeK@h|dGgATu&Jk_#o-4yf5b~FCP*p-(ttcHcmH)BA>x0Cs1 zLB_vkF#dJcd?SbPuldXsfVK$6zn}v;p=&?Whd;o1_&dBp zdd=X4Ft+(TEP#7q6FdL|z-M@D`;mK~7+kO#zdPYM*bDy!WzYgacn>~;GjJQ%{!8#P zn8E#W6?_gp!2LDkbztK?ZePT_2xMG1^*`SIjt$KJLMGx)!atet6lCgd>VM|^@1gz& z8IU=j_n(XN4>^zvd5{kUP{^~V2#To>lOd_T9@g!yQ~J{dXep<(FUDt*MV%_ zb(wDhT~_PP%W8qX4VTqRUIs{~AA(_*856mz9%MKBw4MD#c8I?r(P!AsPLIX5{5Bv~-CTNBhXoa?jPt?BQ6P|;#|Dh|$ z`w#y8{p>96f7<^2$N_kpydQ$ka4&v>bRfGupVx!PUqCqb-0KhzH*-&2;NH2B>mZf$ z|4Z;e05@x>+x~`pl{nqlN3mDKAF;Q>W299MKf>=jxF2O-QKWei_iVO#3cq#u-2^uL zjuG||@oNa%!9Gsm{s4Z6-#?(1vE?~Sxu>C$@#YcUx$Q^~ZXdM%2l;~E(GC5J*}UiA zR{Y(t8rH(sz{WjnhtT?OySV?4UXmmEk`i`u|Ci0sV&W`;>)?903Em^k-vCv88}l1& zRpf$d88ZpRb&9U92rDjj~wP*e-Gu5 z4p|V!@!kvh@EW`ar{OF#aUC^tJ^ioWcjX%FB0NZV=sR>=qvpFWS5S_hfn88fK3gI7 zuQPQ6`|8ARxhZpSLLN-OD)RL>WWyV92j}QjD9q#iW^Yy$=jcCSoUloF1^c_m%&E^P z3$l4O?aGd36CSM zxydc-olQM42QnvSQU7C}I?Cn+7nrv`JWF{fRu?19`hJCDv#9?u&z-mx zCz8+DNAzy3aH2eSvXs(g|K--1`jU-lrJ{Kg&2cWG_0ZoMlIq zLYe)J=S6H2c+ycs#KVYue8pweLD6GNMm~^oX2m*AGoh7-_d7~9rL&(;g>Kz zm+J}n82ypOZSSB%K)V2r;eQ&#UJ>+qE$%XZI2*LyDCM4ZdqfVs+I{5BJN z!S=bz-)7c`n5$f97@JGmj_)`gn5+8cxeB!No}XgP{G)T#2-QV%RaG{ZH52Bl5__E! z-IC_^M3*q!}~8}3(pR=*r&{Yr56E794nBwv=5e5PM1E3&QB_I{zp6QJQY~eZYIVF#yKjI*2{^MKdKfc8n*;~pU zdrLWxJN1_GARl`H6kdFb{@=Ig|9wj(!*8kd^f8s4oz|tP7~}XSrg;9JP!&|4JfWK5 z6LKFqp;~0!r4#gjoIoG(1oy}Z`ae#nkvKl$`XR9Igqmibp#FbCEelVm722@3x4+BW z+4t1RN0__#=yH$`EO+y9ZEnP_9L)crsk2%>OR?+_k!u@s#^`J}o5d4(v}OHzT7+BLaIAY=H0J zcRTV<*bLQB29L8XpIEg$3Rh6KGM8jJaQoT#%!;jf4(~MRga=6DQJBMX@(Zv4nn%$; znQ7_cyU~IAi2kcwOJBvmZXRnSeydj>e~W(QYv>PVB4XVlX)$4esHI9AxvU=eG+Il2EKk$WU-DQ!RONp{wIE@Le?aGo*u|CP!(M_&l{Qr0H=0Lpj= zwo}K6rq3g}N4Z<6`%}-#haxD15-5f;+6|>p0p;L2I)k-wY5PGHxOcXw<^t`C^V5Bx zo{8x`&<3RG1NB8r_kjkGrVq4#1#6tspV>;?t_9ku>$O4We)>Sc)cv|(DaYMSIBVmo z=%>HZ3fv{H{)%-7IPSOU*IPxqrgDZp7X9QAjB{NBC~&4yMmG|-XTJ3DSLi>#h4=sO zEv?4S$=dbVZIt6rDgSu-@F~;$zj8k6Ev+Mb1ANSc!n=^S!M%`0ypJ0>Crm1Q*o1Tb zwi^E@Ej@<(fw420v}PvhAhW+u`plo-Y2yCs3O#Kk`%FUS-+&I%$K;3eA4}yr{d3sK zW9-ZHPp%{E2;&RH`#WnZ>_a9uGlqwB!pE#b@(X0wPL6*!ZDI%Q_}|l3szFyS3pK5$39T_;oIYl-h^Vf0Bk^hQh z{AOIEH&2yYhu$x@;-4j5b3M*nr2O}>Q{L6g+mC;c@sT$C@8CMU;~KSRGNwYl%o1hSIHBym6MqNcT2r7_zO+t%2EQ71S#{C(IeCJB)_&l3_!~K5-K2BPs50QU_^YB;r2V4Sup63R926!#l z=E7Iti*PMeqA%c@!Pq}kqpwgiYq3xbv@JDZi>_6^y-o%5-1;i@u=1~!Tg~X|WFS30Q*$c;kX_L0!!r$SaFq84q&%%{3 z53YtULC5Smbwb;o8nr{q7Khck*I_m7uAwi(VJ#%iVz?1*=UI6>@;{+#VV#0d8&=2p za#Lq;%bn*|RhC=TWo}iTb<1_eE&r|w1%^Fp#NW5cLqCFB4Ldxl-!~x-@on}R+2_^L z|K(NOj9SHmBh0G=Wa5TeCC#o?GWL{(wMt!3OTD#L&i%DY-&L!Oowdqz)hKIO?ezN( znHyHCyaQhPf@*08cvX0o_unS3ih~JO3F(vuYgKlkR^{hw=?AV=<@{Q?W+z)!n``O+ zOtEU%pSwN9s*S0oAK0t`1g<1YGAllPg9m$s7*er_iHq_eoE;HxwxzXgxL%kVbOoK)nx z8H}HDU)_RaDIwb(z;Z*jd*OaKv@Xedvj)0SSsy&rdJw-yjein;rAZdAW7`jmduFor z=1{WrB=#Qz%MsaLfS2LpQ|O`4fBo^du2t8n)e8PsE zF>fnyw36|kN;Pk7R130|G~2el&9^rj)iK*i2xqEf+~_+<*zcnM&!r;l#oJtrW4m~cxKy^mrScUnbgW#e z+)>Sa>Y}gnsH%}Qr(JT-@TnGA7wuF1ilg#4eDdzITMY+XYTW0NFWMzP=>%rD_`Zyb z@y~RtWwuMLhn!X$`)j}8v^t2_Ip$N>Lc0|tf8C3`j72-G-Yqt(Z;wm;#2eT|elEsZ zWxgtvU-GD8R=QP~T*Z8?bn5F>%vbU&x4BBAlu7>CDiwsMTbUQDX!BGl8~+@872|>L zupWwE=|!h@W$DBC78ohyPCC?8i1^He4wu2M@^x4(u|5tM@bN=U5D|=zJau!rmUM`*H zy3LlHma5eN6L~as7MZtP1=pH*@|;r;2o{BdS${bT3S^YPVIZ zt}Mf)vYC&dIRaCo-%K zWGDIR+Tl|0T(wRu=lVZgtzPIm5Kuod{3h=IJ&n|XE9w7iRGgdpzvx{#qPhRa(yT=M zlFl|N+1;oVSEEw18qxo5lrtD-rC)4R#s-^}In}7Foi;0*csbj+pFL?--laz6`*^QD zWU~rSI<2D9jVdNy$uY4( zuaW-mM)ac^)sWYy#`BHxk-ne(1YndSe>Zx3^LK9NWOgMk<>=*^^_fSnid$d3Gy4*~jtQ=?C*Ek$Wm>%4sD# zeM;e-lDf=pr8(0qXR}Y~#LM8m%-m|XvNqbSY-CQEPq`BT=D%_Ob8i)F_3`}ksfc^D z7+JE#r_yac*8lOb{((;wn|$>D`Q$p}L;t}?8^EWUJps8daQ`oyQXR5>!Y5CJOI~Ed zEbjl9DfxH~`FA<30DeugeQGA1mW@8O?(wM&*}g5->fl+{dD3om?elT`cB`BH^;~qR zcec~&BY*w8QwJ{Ct->rny4-fwsrIX6UaVD`?5B+qYn2E6s+i5Q+H+hj;q*^kOlO^I zo(C8G%4_#4zuAuts9)nZ(wDvt{o=9Xl#ie1qhHPgRZ2%@aKG9%`(EpT4zwkKu^YeV?{AO{#BU_z*bj1B? z&+;?A;a8`_Px-`Jo&&L#7um4L&-;Pv<0Q|6VV(yQe#Vm0E!P%5>nl?)IumQvY{{_P zd;F>;Uv-=OybRJUyFH*-?yb1?0QH|Z%P~LBNKlgHKSb*aT$T@@i zADOWz!20(AW$g$kdtX2~djsg=aQ^YnpB-TSPe6rs&OfsF>vn1mSI(&4ya}u=YK(fv9N&Z!UC#~2*|U-h8|Tw4Zb+5aYcahA7}X&23Vt@ zT1_z-R`c$FS~k=$pD@m9+nsK;Zw;{iML?Z91Jr-wtl;T5s~g#~Awc>1)OU&d|4cRO zUj!IP@S^`#Pv6(C6o-u8Q7?zRMhVEo_8KL5ex>ADHR!)4S*aWA(f_HJv%Q}6pX-%j zuczO)URh&_R(5i|a<;@#&et=S zs9qJr^{j^)Z@KL8Ru!^(dp-4Y`u66{)veT-J6F^Hy@9?#{Jc9Fxc=!ATtwd>(odX# zyWeWsN8cW_%%-mo+7{Bc2OZcu=QpS;hQ7f|{Z{vdeyaz2FCVb&TQQgGpT0uUv8`}Z z|D}(UHeVbc;);*)%K-`VgI3~v`b1~kqU58!R>~m{^`8MNEsVa=sbniXoW9a!^l5Hn z{%f$;%HABbayAWExm)PlTtFY^R^~r@dac6k++S={ypQ?M#4p_!w8|X4R{6dGt0KAA zsYQa2JhtK2ud(r=!hyEY>FSzUR_w)@|-ud$w^W>YGd(eLYKLns@K5Is@8WTqcthPe~Ry%qz9eG_=C$fv}gNs<3;((j?KgZ7)SKq>3tDk#g0CJPxQy%s9 z{HYYHV24+kkY%SnkIZSOzTWJm&(dL)pY^IDJi#iw=v7fW^?CeD>|Vw1^2$NngniVP z_IepO=6c#edjc7^->WpP5$7x~>!rWP`_jw%)XRKoFa3#Lo)=!VWqH-!Onn{M>GN_t z^=dBTcpSBA#jp3YSAEpQ`q}RQ?TM};uY&k@CsUtscvTaVV!3yFRZE?!ZaBfJ-0WpO zd4g3%eyXYCH=a+he5VsEKQh36>$jv@o;_Z9iPx}+?MSx{YL2E_?sKVD)fu)sooZDE zJ#vBnQmPdgPPH0OdgYr+wY*s#H9&o{M;_>)EION0t+ulswL{C42R&Plnp|ErL;nGf z2B7z}M|}{=H+k=Jr&__XR4c*W$9uNlinsSzj{W^sto>HSnZ_gUGW!&B<{t8*OHcj% z8+PkZn$1eNG(C4NDcEBrZ|t}7cX(6)xf?yogX|TYH^`ioYGrLmwaRxrf8J_PY$VXSvZ-y+tfiVUo-ITUdP{VWjcgR2E{}GbqQfz-h zE?&d<(>B(#ogcyV7ok@AAll|dsC_>=kegV;4|f;g!87PU&W=zI^y1&QC_?=UA{dj8 z&;`=T;r;Oi(!2(i!S(P<*Z{TAawB~gKE%i67kWvggsFz&o!l^iqZi#oeXabP>QN_PCy@ynRMYFh7L(JG~Y(L+dr z>s{Ya%J>3bxscn^J%d8-(Yqi>S% z6XY>Cf4R1QNd0ida=qErXC3D%~u7A>cJvRn<&xNEt5;CF>FKWDkVL%8cw82#H}`Y!gGnB_`ciB1putD6W@VN4kF zBbKW;Z@Ef5%T?-(P}z=mR33~_1!N{Kr@gV9<66#o*J0>agzHAUW35Atv3z3-`32mcvss7!8fU%vSNxXZ{tEZu%XZq+$Tz7kervWIcg#`3tC9K% zX=lR^IkpUr@fX;i#qLLTz*Ug+0%J;T^rKm2>==;&s_?{F8e&6`8<)@AJf&2?*W zJ-zuX`?SUL{V_1|6#0A~Ka*GTV_QKVHpIrDzX|c!9gx6SXJQ$8&u&YfC!G?~4t!s_ z6Z^N}Mfe5S=10qZw#kZJ6wRF7XvNQ;DaTodm9Wsp_n#81qz%zZ-Wg4M4*j<&hm|(3 zfN{%crLRE$ZErMVdbI!B9lZY%tel|3%H3+S@_1k54@arsQj`ikIVw67#W--3N=`
-k?vms$cLI^uNT<&;M)q@p-;y)}XWP zuMy{-+(W(}iSzQ`9J?B>{qP6y^Yj14&MLn!aX|TnsaoaNe%G&Emm>!T`7e(1Uvw|S z7`fp3k#;|$?xE9|e^(7Z!8(3|WSrBVoZ&x-etgI|KZ1|pKa{(lAU}zAd&LIz=UM1J>w8;JCrJB7 z;p|JTVMsQJ>~(tN%~_tlwRHz8`+jF$weBo#(gk+xQ)%lhVMQ^vQFM#qZL8AF<7lMdpmO z#Q%Y~cJ4!P3B*&_l-Y+kSHKPuGND zauk!up|j;@^?#A~Q7HeHdS7Jz{}W~E|G)8%2K)UT{vQ8;f5JcGUom!4{=oz$F@+4W z$RWS}|7Q7rlXr~~dh~6*T>t!=G!`*@f}KsqdB6WoE+H-MKgj>YWSyPMi|fMr^0&*K_u)4%fapnJM&C1gMS)9{D(|0Db{{se!9 z|BkT>);~bLpJ%r`y;t7eDUTyp$Nn!?|Cg%&$>G!L|2btV()3X*{yO_#8zBDQZ1L%9 zpW;9G=hFWl_zV0M{u*&Dp?k>pNBvd4j}NYdWAyfxRpA3<;yC{U@+0WTD6>BM6Ml0) z(YNd#?USr@kM@cGTIxe*-Nn#Vf6=y!p$EO}tR(uzFM4+u!(U40WB3FS7qrT-5*ZQ~e)54O7SaC$|9#x)YqC>IZ)kt|EU0*J5epC*fDgU&FI-BRaQu{^%}uon@}ey(FtV zf7jRV`hUYdzlrDLh4?M}Hhu?p;wkfw)jL=E|4Lk6vG4Dc`rkbck;C)yKbil(C^=eN z5Pnx0zmGpa`}j)uLozY55;|sA!qIlIcjKLSH&%Q8?hVdQoz6C4xU&{OiyBjP&HjXo@W(7ljpOknZt zu9aMPO$eVOzkuv}3d5u1(RZ+0v|;Mcs;@7E@D*{5>WpKrD-2(yFTM0)_!_zJrx(LF z$TW|0B$MFO%BJCTEYXGhqaQ)@JU96>FjogB|!MQH4f0+0GuVCY% zXPQkr%eI|owG`9O>S|0+qE5#rRO$jBekpzFSp+-@oKyl9mXaS`Z8Wm?>XcA z_IU&Sjp#q;*@#P?_pGq+lEUz2@r%!06W&U`4R_<6csJgQLpXv)+>86L^rJQ5ezNm| z{*QIevBC4-D$VWE-s~Loe)0kPJcx(!2riG4Mf;&cc z&vDQ=$`FPzg7{zN_>aL+bk3;%)JHLI_Q+E)Zzjn;^q2W|-y*L&&k)j>Ko+0F7cjP2 z{@m*Ml9Tf56q#S2OP95Hqw6XZp#8uVUAR)@7@Y=2)(MqjJ?8sA%8 z2jpUn=pm$emSJoDU zw+g=vdAm}*(5r2k++zHRFL)Q<@ow@L*8gi>A8v5W-FPS7jrZb6t7m}27&DGP&X&%P z?^m(EceB5d|IZ$qdtjOW`33&xY~|r`^8>mr`Szz<&Ht}s|LX(jnrHv7WB;SKg#BO0 z{%6PclZWE@(1CqPLuYRik!Ql$;LfLm@pYrdNvfVe6BGc0$Ty#uh^#6N?Z|^kw08?|G53=-}{Vvzb zUF)=k$E9B+Um%HI#5upKhQi^sxEkwm9j-_AMb^C7BJY&u|%kfIQ8n4Cc@dmsRSNdn=#WxG*jJ>>-ycAMQsi10Epr zK4ivx5!<@ahwSG|!VsCp1eWyqWXZVx|L4fX7x~u6FX2%febJh{jg|KY)UOoh0DeV$ zoCDaQ4)`j)@um6~KgdV1p^$&)T_x~54wtbZBJpQxw_nO%0 z$gk_GPoPgdAII7sbB@RH1TJFLux}ApV?FZg52)L(tUpk#epmlz)&DsRNNZ4E!jQJY zur|YpbEJ^2)jxsjq;WlNz(y%><3lY!2`{(p z)p#xP?GMl9_4J;z-pvC4g7|-chuG7zm)d5@GujHC?ZXGxg-^QQhl;HIcy3*I(A=C) zY%j7V=!1v^aknq2_Pq*F}-Yd;RID$sp zi~Ddt9>DOKo5K(WlP?SdXKo1(ihCHTg6Haszd3X&zq-nlC*`h%?A@xYNhxzO+CStY z_U%A7`f#QE7}qB$Jhq4ZznA@gQ2+WmHbKdyv9~NeSHS;qTG=}m6*Wy@B=)C$MFO%qBCiIfRuII z(X-k3ce!!nQsdvoj{9qje-|15-fH~Ywqax8BhALHTaE9Nqrzh;{s9?d*yN=6sRm>B zmDUF^zduI~mzIWABl0w^#(JdfpK|$pic{y(g*%ypN$59ue#PGinp^lsPIGbr9M$vO8o zZNF0+!_AJn1&wk3;oA$st@PV)J9c1fulDyz^(!VZh0JN?_Odn}2HeY_wDbM}%HS*h zU+RZZ;qI-TYguV{nY3PxKH>i6XZsHQhhcN6{vBg9{?YJC+g^>14W;3=eejs{eRO>8vnE3h&EOVY5HiJe-1s5ufvP^ zJ-ti*ex~nSnm3?v>{36#(U)8mHVQunHz8}T@y+DYm-$9~k^jW`RpA!lTX7q1M~Ct# zab{K6L637PqyNv#=r6}p<{#|R|F@6*iOcJV_tKNNypH(Qj(H_sjo0G!cmv*uH)Bkl zK8^`YBESC4pgImY45+6D)l);{@QV7MObMsw)c@+T=s);Y>AVehNnlxJKQV(C&byW5ef0#``Oh8m78p%EZ+`f-M5To3(o_yWF!wEIlqQTk+;XVbPe ze1-m1utLM8_{uG`@dehpWg0Uy@|XTx1j%ocZMW-k+06 zO1v7;hc-XHr+j*?aCTsIcs+USCCaC-UlZO)e>2{Sx8ZIq{ps59PV(Kze%IXGkFE{x zr7!SBIz%2pBksj;MM1caydMwXK}6raRBb_cn0~qc?Gbthax2P)^U8=q?+V@G`Y?o} zFTEzD$q8h!B(HytTzKx<@C9;_E!X&A@9QVmghz$Hg2QaT==XMn{ntLP%`Yw?{SN2( z8vPsi7UJ3;J?K5I{f|D^*Y7&N=l#CH_V3_(_!54A$M85hTh0HE{R7+e4{S5$%`VN) z|L3dU&v$*Ggnf@8+lDbR!}guy|3A&XXZMb+n4c~@(Pe%*rZQ~b1RGepF-LaRv;WbJ z9`qu4g8koc$^ZWe*KiT5Mzs-eHP+)gT#vC<_J5N7pJ4y9?Vs}hLH!T|Gwgp1VHhJR z_J4-`PmVfnn0)$V~!K83vh$)5H9$Nyya z`f!{5vQOuKD89SGbHmZ+>dO(|SLV5udyYFjM|5u2|AOwSOZ)XoJBhv~?+1k65 z)^Fq4Owltno?C%ugn@0IBhvKIBF}My=ZNkW*KOa~y~ z-iEvJPP`k#6-D8_532u@>d1usGUCaka39%U z$5)RXUB7&3{l}~|e21iw#xZq3qj^PRrm5DkoT`?R{qN4gGHV{Ib7=b zlPTdeIcmR!w_F<*9x?~u8FBu19M_U;Px9}x{d@QU9z%z? z1Rkfyx%1sS*M=wP7tt@=C!8dEE7pcpWAY`g#(G?_kG6+!o$&Rz0eSzoU)RS&e-3WK z&FEAIcct`CWPavfuuk9pHugXIxAMF1Tpez)->tX}w_^uhhL>YZKiBx||HJ=H-@xEz z_P?}-%h~^As+j#>%Kkqe!Yie*!0-CiJC!)wXc;|+Kt-i){6ZMYln#JllcEPdBA zXX`%s|HX&I$F&1eC&kkz)%_FN2#xeC`ytMWyi)(G``c4%!+o~p=l{K`Aly&y*k=5p zK;J*TXP@W4!8$hdhtXf*`5*B75!Y&XMBH#jc~5pDtz7RThY;(a3Hf|-ukTHG0&(mo zcSbp}%XhHCf2P{^kNo@>^5_3=d-VHvgskH}hcDnucobj3S24C%-=B8&|8@P1S?%9> z>c5l?`|#qYOKd~xE?p)|2O|{qkW!(n{YF3 z!L7Irx8tehulzOVxt@~05bOG1nE&^4_o`i0J$OX0pCcKYVovZ>7gEp}Wa<#_g{P;oW5Ht9&mR=RX}HkDxK~-|p0o zxFZa=mWL-iueXW27xy7O>;LVk@^C+W@`C@jFPDc0=no=SvP1d1BV_C|g@?tp&y|Nq z$c{);dFUqlkZ@eLbb2!7`ro7*<12S4e|Lml`z6sS++{9*_r4vW2faw55B-Swf5LHD zd=6j0m+&aQg0Wq9>VGlM-(3F5gXZ;{@1HT>KTGC>2TtA@1~G(Tj39+HMsavt`HuMP zzAC+i!#lz^$i?TChi{RM`go76G0y+A^6(wu@8Jh{43A^D>(1~s@(EnTstNbm<{aYJ z(kFNA2ot!Pz8*Px78&&%j~lQN9sBMK3GvUN_cVAmxQTu<`ss0PhGes6 zg@>>g02D?V&e|&;Q3G$$tK^u`S`nAF1O%yC%GiemCBUcjLV{gd=Fgy|@o?UeWzz zr!u9>J#@SNEB`;)>RiInN9FqXTO~A2~MT{^#7k>zl+>llyOV|IP0I zxcisZpnDy{aH0EGj-{5}KSr_eo;$pDW+HnoBG?~TcqWtwW z;S1!_zw8K0U)~YEM1K^+o^yWwqx+Bl9{7s5uOdaC@Qfx?+6BVjK-RV8s$FNob-Kn^ zikrZ<@Ev>)Kfq&n9QpP0)M4GouMeD5_tdL{*tc;ke4v?q*~-2ohiw~CpT&7^aUR^L z@R<1V7Iolm@3UJ^co3#B-2VXQp!7`H2aniZWhk3pQ}F2l(4@yseg}~yP(Xj$6J5y$+$Ep5PeOj$e3=n)Om{4KSixN_NV%2giHyi$x-{Yv-966op&Sq z3i;c*bR8$P|Ki&!x*oAlXMK!U z?=O7H{Qghrr_TNCwi@;)d;ZG*?^5hI zc6wa@|7yqXz{~J*yb_hD!d{FSS06vfZrRU{WEV~qvl~m{BwiYkm0SuBU~Ei`Io!Jk;_l;S)7w;lT}M zp{3=w%~#tNKDK#VcmU1y<>8}6<>4d8?+o`Bza%u3Zx0{FhxTvgzioZIuqGV)R6#iI zJSWhErSBGmW^#d#YYQ36`PL};vb4qcrxk^^HvI(jc4glq+QfK`AV`lvY&+z^>gux5N`dEwgqaI+Z-xX2?Z`Z~?5W1w9#2``_ zM+VVP?~xMcT`3Bm!ja84h4&ZU9!}bJ3XNUb%VpYKrMHKM^4|&f9^Ya7@78eIwrSiq z_~P)PZMTIFmfspqY`rzyzxlTCk(sB3qra!@Q2#to{43#Op7%%h8vm;_{^$Gp#PRiE z#<8=Q!#p0UH@}Y@Q#Xxgj-S;Og~7U_FtnSm1xD0a(MLH=j-K%SX7o?#L+Gxw z&fo!aQtX>Fcct&3bp|&Sg?i^WqP#Oyk{=^FBYYMQ%AZ;N|HG}<7-K06=fs`I1*8+# zX#ZXlR_K%di8rzA-y6>}|L@nW-&f*3H$?x2Yr+%?GM?$&HKCAPhYsVTMdSt~9M>(K zp3F6&ShxgzmC7|XQ*X;PA&F9Po3I&Muoc@-hNY>3u$^3Z-}+EaF21H9>?C(#H}+uI zbEqIIQH8xod!BKOVLv^;{>MJ$v~iYW`VyYff3nFM36<7hoi&H=6kj?Va7;&?^1rq) z9HjSL&_B>t7^>+t=vOcF&8V}<-i5+YD{i>zs!&JPBihvsA18#HkX4`N zFog`3J>z}io6&++v>|~mbQ-(Q_Yb7>51doyIwpxedjA>Y`(5mN4DDv$W2B0Gk2HN0 zW3}cV>~bADUB_PMrDyi=4I*l1tB673#=(G~hT+pb5=b_@Vmu#p+)6^5Qes zh8A(HXhQ;5#t%wD=n_ui=u~M~`nWYXjLpUMw=_G#v41gM`u1mslR{seiqYuf%2|iZZ!TJ9KH}FF`r3}&@o6g1^Np5>3v9|7X>-_e|BRiB-bHr+`Nd~fRyp`i8kZ@C>Ab3HnB0}x{Tc$ zYtLY(GzU>?+tD@pbK?6(T;FVyxXw++=kW{wCnzvJj{yu~2*VgDH1@rY|ADdn(K7xA zMf?wnT}P?wAg6>gXZRnS<9|T5H{BREJ5S^DwC`V05Vp`ej^AiJ>c+5@-eaF`Y@?T< z-*J5l{vmMQzXY&d-0;?op`6@_==ZRT+>Pi5G2tF3u}8Q9*&6NgI_>l}*MZI%_c!PM z7Wf~aw}Srx`p}O`$5df2_F+E`;2^3oHZT9H-}3%tC#2h={g2E2y93)i9}Hm_Bj@G+ z743g=)N$?Jb&YgtQHOeT%y|EErJ;e|W86QE{U4{FK>mN4zWL-C?;rIC)T>p>8||MX z#-N(*(}GsCA+EobAdmfxy!-t%p^KixAX4aT(Ee|}vFZtIz z&-w#zS{>%3yYRFS=EBfH&CqntXUP-(J!yR zb@aLN?l;xHv+_SK|9^9gZ^t;liwW(#Nlbb6nR#U)UyI!7(lFp#9>fqoj$w?@^WS%- zG>oRS4ZD0l=pOg|pqHLRU#;&4hZFKsQhp+fTu=W0LqA>ff4QZvPL@tqt3p zrwrRsjj+&oTpH_rSgA){Eu82`(Hl4GX7`3 zr_}%Z3PPJS66iv^_FOWC&+2bcHxJTN7{^omV+Prdi1Xt!ww**)*&Y4YPti|f8Z*fI z%NP6pw7+MC=P-|B?1eb~9@hjuBYYO;u#9xMJc{%5N$u^44XeWi`U-O6^7f!SPEHl9 z4h1LWRsKPRHKI})<*#7}?>FwGd&r^SQtqKQ)8}H&5U>(6~dPkjpRn*YqoW|~o zRiTz%hq$I(-?BC)*}K#IH?aRtu#?IB|NrPowtruWoy=yha7;ZKa2zMlgl4p0ENOh- zz7vj_B&UoSXX@DKWR875(8c~2KV+db<9g=0qO1Q!x{1{hRfH7b7UN=opOJ1 ztTz7lZ&}kRQ*nl7MYV^zbo?x&nYw5MEU-~v)TtZ;T%n9#&EfQ8Zxz2|APMiR(jSre}bGC z*FS-&q%yBTU9h4b{roC7>HaI-KaK|T_dSE`Gd+WM@elaPny`3>>v#Nu`F}}r5Gjl! zgOfOg)0oB#W-*5){{zU%;|uTdjR?p2e+%RpoW(h`FRu#AFaQ%{igq~S9#aDMg4>B@}KoT>rank`+c?S1hh#n_OV^=A1IQ>QQyM`vKV8h z)$fRP^dzR}8D!O&ISj00*I=kb`v4>K6yh5HqZm%D4JFcPG_Vu>^GfOQ-*|ESd=q_g zTH4r5-+~-Hiwvf)Rb2bBHUwFQjuUIccCs9a2KRN`ePO5YF7(m+(TgPV{qOsJR{z$C z-!J~4`zQNb*M{AW*@FsHq6&Ml5Bo88-u-KbAl_q)ygl5xD^@_5EJ@1@T6_9Q=p+DqTy;rI>yDPI6_(f6^z_W19==-b)& zihKC^nLi*LeLM611J!H8ar-QNUH=T(gl4p$6>Ui1=%;+cWbV1OAxSRYrO$<2c$fD# zqpeHN*Z+s>!Z&E8G0Ohnovibq@O}gUcJ(PZValc&^IVw#j*W3#vgr)a4!K^4VQ=;k4tXF@ssmVIB(@UzW$uYWtJ%9|3tkcj4R~=W(t$_BOD^*jvG+F}T!> zGNMG^A(@^tuDM11(Y#v0=NbCWJ2KiM(mjhrO1X1Rxg+g!!spR9uiSC1y$hG@l*TvJ zg=6ojGq$O%7VZi0->w(zw}M#47kr*gfJyNad(;i$qVGj+i}HNC_KIywUtb%F$PFk) z2}-f_B6DD0a40Mo>)#|C*GAhM<-0=ILM~`;wx2EtTj|@-@Q&J0Ms}Rkr!0Ouy@%{R zRS?SQI}z6a=^Ir3+oxBZ)j4119UKbX1&2ZpdXYpQ`myXjb~$c0_Mie!ng6%-P#9l0 z6ecikjFcS;sm+I!uZO~D@uAR}yvO+SJ)s*t=k8%s z-xHF>_k_L;_vpX9CsaCL74~8u8h>bglCco>(+^GFHRXqhD$EMcVIB)OgRD82i|;xV z&c?X0x^RwM#(7-83JOm7{;>{4So-Xtuz@T_3F7?UQgRbEV+$_N|J_QTl+PyA_u1zv zXFYFTKou+mVpRyQk$}{R-v64d2vvO^<#E7JjU)V%y@M z-WYb1OFvp2_K=O%**#{y<>LDeg$m(?!v&#|tioO_efw9#K61FaAnYd(;2^3I*H%gG zV+YbFCyMMdtAP6lJC3NmSmspSL%QDOAmUt zs2eKO5tZt{J?c%z#&NTgz@{$ zL6#oZ0Gc|lAD{lo3-1XJpLvhT*K6+T>gcQ~@?V0izTzX-F^o*(sQ zJ!ji8&f@|e>Iz|nZ1)eXN4$?V6nw$_w!w9w!#b4-|6TiZi!rUHVzyav=rZ2dS-mdw zmp(o8o!!7jyEG0sY#!Xu^0V1T`ik@y&B$|w(n&2mQ(sI;SVu3y1{5P-&#m+QtgH?t z!tKR{p_FX=8}_5N>SlV!W^LZG!mx$jb3h-;zQVATz773V@{qVBdaDaVnYisZy5^ct zPVU6Q+u5?Zf10!y81LnT>-z1W8?{o9@O`e>5Q(<0qA>5l88NjOiJH8t7a z`|WoC2T_e0)FS(%Yr^<$-=Jqa>AEJ!jPMkhb8XqW!Z0X}0cj3PbBIhi$B4MmT5}il z|8&l-30?DRl#gpd&t~O<@+i(V$om&6AL^W^9t}8-C;K;?5FXoR{1Ow|8S!6=Q*47w zO1mY)J|G7Uu>Wzz|7p9n5z-s9LD2Xq&;38HtQOrCL#$jrM&#B0l9lWk3y!7S!5j|H4T z=Qj2+`?0%#eT*yq>FmoZ{{NfVx9r$qa%2=b*P9UbSuPqL|}$c%8-S^+r> zcdZX+o#z~uaUSUzY0Rw;7wGYyffFn1!wNn70(G9_voQ_(OxY*eo$cq?|IrSu(0*%P z9}50X9zdeib&tF526nu4)sC?-`s!UbdTmSEcb)x)o7RUSas$$d^`V$7K`QCo3$7cb z!kdsi%NCz??(?pJ-k!KRY!<%-Td@rt)7DQ!8NKJAF<@+`m!rSj8k9&@UZXG7|0mXY zzUVG>oyD$elj}ks`mxh7yRaL3P=QKRVK2tC;l^jxG3@zCa!NQ;p)NoU1JW48&}Q$) zdmP!~**Z_!`9}3m?32cR96-DB<{+6!6tN-v=kM!$>h#~&i0eM9EJ81zfuw%`_Mv~C zKg%x9eUI|Q^KbXP)yCs@t_~fjqvzNE3F_gk>fz1uSGoK}_MyMnngdmAfCk&^ZO6~& zzj|pj;5bg83C(ChE5`P-?+w z@4tAz)zU|W{Tsh2-|LUcuMewfu((OybJ=ud|-9yqLfLLHCd06WTw{ld91Fhf&l>quo8$l6C0Nma6v5>*)!4 zcd_fLX3Gm7N4{OwqzxBs_`Pex32{wmMhjZeh6K8h#2``_$CK-?jPTLJzURYis%!M! z_4?Pm{^(QU7e9VgI89Dt25~OhEIEgHEZ_{z;vAMSEYF=MFZ=gjps%3dwEnYbqo0vK zSJ>$1l4`)eR=ZZJ{y!>n=Y+em>@Sh?-q1ySrxXDW$0Y0M(K8Xuxp{Ymc8Go6w9FMBmQTdA?@!$u{5SociF~&(` za_pq>bxdFqQ^+8T90v9pzm(39^x|6gBb$t0I!~OVK8p4>n4ah5mD{?TllVUF(I7;k3BnmK(z~IfHcO#xP6HA(gr@OxQMwdEo_QYmNJ3 z3K?{|rmhO-sC54aq**I%^wIlq#xZAc4$C-?3s^zHm*oFB_wSk}T+1Y;&bn@7js55N z0t}S6e+-Em#)y4VNYh8rK7VNqgM2+v8bYBo;~EAD_qUGTV|zDF%11@QaqWUW#I+0J zyNYXP#lENPH3gyA_B6eOEJZA5VxQCGX7dPN;eGKR%hGdOz58vx0c^70W<=kMEL(pI zeW{c01Gx>k=WS!--x$j13oqFiwv*-9iCx%@J!qfihejr}Ei1_?bcowa?nC^yPW%^8 zQoFbp{qj@%$4-|vXeS?po&wv~*-rMB*se|8M?d=Z!m!`*2XGM8IMT{)#$k*rQ^t^A zCydNs3OQskxJes)w`a6j8yu-EeDKAM7Hfl}(=~R@I5&EVl&i&TS#dF5b|KB0M{(2?n!LT=DEaN4$$u9a+`D-8AGPh%Q0=qObGmaYx6^d5C@H@kg~K9BfMyFPY% zQr+82E{J<_|Ir!Yvp9!k1;z8wj=uRr<&G=a{440efW3McOkc;Zvz=j*>AVF_BwtDWCc1-vT?~OBxXIk z)9P<>uW+3Iclp1{z4l9v8-F;fe-7Oj`2V4op2X$#{|{@U)vsvZoD5-~G~$|P$KIvS ziN3+J{J+kBY+@_n|F{1Ck}>B4(mIH0#Cb6_WG(8j_)Wf{WbFTHh+*SV$79$xdxDJq zZ%t$~9^ePpLSFWNYo%ZIe`}*B@bms}UE-1$MEi>TNhZ#j|HoDxr)O{y-O7SkM)V?< z5wUFO$CLdFPTB7?rjdQ!HDQ)){4D$WxqN!)^H{(coW(iBv4<}Cyc<2}mG_g%f&BW* z+hhG(6qfCG9v85Jf-n1iu?}Nf<$w9@%J{E*m0$nGyNrGw`Sq78uWd8pLdl%V+v@vk@cp4brL6vb2&Wx4jTy{h z4)a*R75|fc>K{xYgDi3wQ1%Zde1C1eKa9{*N#9?S?+?QliozM`oW(gTBV8yTsvFPK zC-><$QZHYiuON3odskehhW(3zY2OeZ{6caaI(G6WCO05apP?ug-ej1SGQF@ zR&FFwe~x{OrGGRBQ~RgXKAX|_5_autR);P0t=NV##C5;7lZ)ofmym}Pc{Pd4>yTuXsnG`R+$H~4x=!@$cU^U^Yf$-u=-(8_|Mxn6ANJz_ z4x$=0$glr&Q2yWV`4@TqC7wUB^jx9mKcjxemH&U#c>Z;sKN;s=ExbpbeX+U!*8h)w z{UB94F9>7PMmN+vfiB4QQjcx3XJ|4dwkXE0h5b@!hI6{-1*F`SpL?TjMJ{ z&o8eDSNwn6cQ5kxZ$drA*6$&sPkXXW9Z&C{UmKdF(S;-ik-|7KIEnoFzcuXtI{ozI z)Is+3Dfaa=`SN|T={t(x0`x){qI>etNmyxJ;U(%23>Yf+AfW8gx3;A&l=jrTX z|D)UW^$d!icK_StKlJ19X>Co!`N1oW&%Q^S6a|l}1MvU#zpaq}Fod{P%+LGZ#(xs7 zlTPFF)azd_2u1Ytpl@W?+OUB>+3edwF}(!w-+|d}zOe@18aBD6_L{Y!)V59NIB$%Y z+=94bLeP9j0XZzE&DeS-aFV7E+uPG15jIq@UFMZm)ktkoP z93UHT94F9(W*oiaj?hB3q74Zw8jtKE=V zm;1dk`W~lZ|6{qd>gE3y`M*h?&>qPD|5Kjr$5HvWdW{3-<$v>zaF)LGI(>96FfaIZzG>mbzX@TP zJdX=lL3@R7@gLmRzuXiG$#v+cmG`Ta!}Ol*?rqzuuz_BT{Qv)aBgta-iVtsnZfGie zUbw&LdEp~-&kG-I*czJ4UJxEQ^@8xR)VA>Pf)|CB;unPno6Eu{w!AnzbnwNYwWTaP zT>g^q$+DM(YS)mS-C@kpb<&4tc7zh=Da9t_=5~av?HSvrFiD>{yCZD2?L&K?8@kS{ z(+Bai&{L?Nyhy*exoy4v+19_)I9KNBp>2V$gMHfz)`UmsiB|LDn%9KSj6W?uPWGwu z+e2HS^#{)0VJ)=su+_fXP=@UoJo$_;VC*bEPFDAfkeW3vhH~3>;>f1W;r&}Shh6mD zXgswg9IJmpI9jqLG~nKXt>FU~wuU{nRpP#aZQ(=5UmQNz^rCR0bX&N8S6TQ-Nm*$8 zuJK}hvJX^k4ykyJhbzU5Pilv&)yljR_@fs zy3;ebGxV0;$^N;M{c|V#=gzRtG5c`<2T_e0)S?b!_8Tv@{sAU8TmPWk`UfkX`S~6E zA9jR+I_n?owEh8xF@n?{>mS%}wCv8%UVlfZmretY;{-Yy?g)wIJ3_bFGV=^aV;j~RN;vitXK z&C=)4zhB?uoco_~{|k47d2z#A%;P7|Ao@O?CC?%L!)#*Kb2HVSDKFMFckL))ld(ySV zwUe^nTphMKrt#_eia*ZBg1#MN>gVxTN2_b1ul`h_y1D3*&st91HZW@pcHY=4Ic&d? zT6H_p^wE9l-|gyOKK0!#KU@EIoc)i!IrjhTRiWHDcA~;{>>_t#PYk0HW7-ho+7c7S z6DBcb%prp-{~t)Q|Me>jVYq?)-)j6pJ0-2XGFmN8TcAohd$AAuk#Q|8eF2U**c?&ED-+^)%V!m?!tIGz+(&71?(bgt2=5K_~QoH0b|m(f@H=|A+d- znuP2X_Wxe(59Q_%hB1QFS^h;c{ELw9KTP;P*Gu&OKkrh$NJzU2@gHFE-(QoPy#KeW zXX82PIeqAHZSM@)7qoBVz8FUa(RPpj{yItTkzeBfeooO(qyM1%BQ9BEtPfqTxwDCH z(sT&jZG4mF`6!`Ze@>sc>5wpU@MmXX9(NK zvZ%jG{?^C7FruBS|7Fp<((U5Qu@k$n8+*`xMqkBejXSp#gbLwGbTk!&DsnG+8hm3& zRw+L!d|SJHW8?CEKyV1j@?M<vQbkX7+R|`}(+b<=F7!4eaw1(LVQmdTtqH={eN5st-};T(ON^Pd4B70zAf8t571wz1`Rv(0y2>Vs&nVw;OU zh0~ZuN2~r1%+Pz9*yfq_VU|9J{$=+2arS#XTb-O2w}3M^izkn-ofAH)A7+_6_T9La z-L*%fuN9gh2Flpx-y`WBz#ujYFaFe;aD})Y&T3B_edg-0 zpMC%bQH>hZq7LsuMg94pBMO!;`#yG*$CJ0_5ZHxc~fEdx<1g$ z|1X`j-=t?0+W>R)dE{!8OURt^EU^&NlI|ID@ily5$a7f6d0fB>3jT?`iFGK#1}uI1 z+R(9I-wM41iG${8lkG`+KeEy(MLYknlrz0>o@7TdO>vqk-j?k@EdddDyIOZUhpB4kLA3pM+jkEtP>GIO;}^D7(R-vF z`vCUR_aXjMrw>VK$F+9z^Y0JZXTSSDAM>@Yar;g< z?#llUlG+Ff*ICE@TXFxV-2b%uCr9SpKbaODC5O)$+n3I9oIn%Oh4$I7HZ;@o^B=dH zmrifR&(D8s6W8uNCCDyx>~#I)QQv){(tXvruU)Q3Ig}FLx7U3M_mWB5J6qVlCw&*_ zK`)Z>Z=ZbKk9uSEhcWKhBNwz0K5YKI_nr|xiLq0*O|!Ad$$9Mpdd3)k_PlmN>(9pL zjnNNb7$b=P{zzjK!@IQ&q;ne6m_b@x3bXXd25DoCK93wdi%hd~VL@Db5uX|IEIR1t z$Ymtx@t;3EI4^tw`E`YaduOyk5bM+h{Sa}T{YAun^A-Co{$mIQ|5KjBIxM}!{Mjgf zq|PLZqyA(5C+e~|b{q4^(NG#n#Frxb^wO}IZ2YX})ms|2(6?e6%CH^fm^{UAkze9C zpTsf#h`DKgi^!b5G{=9q)VG5{3}Dp0Y3=hAMiAHJ>PDQ~---S@-!78q#ZKqgh27YL z3RI#Bd(nB$`u@h|ySEwtMt)uVR%6V@uKQ1H44dq;5BqTd2T_e0)MBj5_%|l3=RfJc z=Tp_j)sd|+&VJxV-@oziY2)9gjDIJMe~%mgMw&iqoPU^JC!KmUV9>Kqm)Zx%g(ufZ zd;7+4f*#lJ&(#@QM}GbP4aUy3A=+2WbKd1WRPg_+vR%EvXWw2jDcnc)OSf4%EoensOcPy5Vh~s6-_~m9VM<#tgM2%0 zR{n2r|0mo(Io#_0$&_%K9Mu*ZJ|`@lab$23=@^Go^vT`seX&hRk1P5 zh->fK7-q>ibkOI?1tjR*h3>0}AA|5&^a=M1_ZGWPvU8I*IJ(h;UL?_new=g6GR`CW zCgT9)3JPcC3yd9?|1lx2PsThsE^i_$U*s@w#u&WvV~EWE-;bVBPNZk$|BSMw%X?$1 z9LZ>-Csv0dX~zF@rjzEF(T{#p{YFk~mHuYwmrI|_B3I1*ILnXO_Qt!^zuJnW^o|Ad zBE)Z^_Z)C<`}j4}x1hhuy@^Yrw|aHhDsCIfupQ-CxZ}pKlU#hMwH(OZ*n>D8UqM!) z3VX2+M>e~M4cZFIfc?UY-(DRKkg*+ckX+J+Kl&DJChdd#{QKW43^n3wQHOdo;5bg8 z3C&m<(5C-1AHlD03@yToPv_G^wjqHo#6IODIfxX-k)MB{?#a;On8XB5(oZ3`q7FqI zBai=z&)eVXm-vsno+f2RO4%`^3^}bVIms8zF{d$&8RYvD2E8xL3Xk#O8^;7D+0;|} z+26(hvShAQ{eN89g(0^2FZlo2r(HcaC!KjjKePq1qvDeP-~T4O}9a+;>t4 zC2g_AmX?TdJN7F%keORF0Ky`(d$m%O9@_k!%xAX%3$X|lCCY#G&_g|cD*x#<^pSmfq_xG|EWuJNL)Qai zLl(IL*=Ri>{p08v=~H>yaY6b-SF!Yoy4mlTpSMu?&)hUu`X*brYnZK$!JjU% zu9pAQkR$thN9DKln;Y~ca?`h8+9CUmy&fk|pa+9@g>Z^IjWg)QnitG3_?>z@eE?^1 z4udBz2}5LGo%l#bXZv0thmrJCaSEwO!#Iq`1f*j!rXbosF7F!qgYzqcxqqViMuxm5 z-TP`UO=UOI{y1s>xi8uW(D;7ekN!E{JfyC7!Pt)L?PsL_Q{up{%t2?4>ZDy!-hk+= znu;O&r{NH)P(4lC7Kc%bW8Qa~`!X>LvoQyAF%PY0&F{xiZQ=Gj^WwFeJGG6w(B0&G zfcfU1FEsz0Y~Zf(g8AoYrnmH&KffX^%;(PnEW{!lsMr2(yDTiGR~2i27n)~4Uy5k& z{KKe0wP!42*PzcKi(G-`v*sF*IcTCE)o0PJuVS9>$Ya-~Z=zeDK?nQJQv23VHvW!7 z_;2SQdgpxi6rd1Ai1PT6{+~{BU6@;E_=I~dV*MDfT^p9%PAll=z7RTLl z0zEi|=f>$X$3E{}s$JWfFU?@2pRh=qR@=TaOB&6V{>i!l{e$O>f0GTv=HAhpdi9gJ zw}sni|B6k=uMMa9eFnWaP^zuYzK=e#|ETvEpr1vpKJddC-VxCpf<060;V3M2pB)oQ z#@laj_PdR@j0rmyygL-ndv|zt*}KB_oGZh&`R@uv=iU{zoVwmP*Y#m@)3u>c9Wb|Y zQplS!DXhWT`YXdZ<k01=oeu^R5dUX1^;GlwTb-VpH4IGU3;f&b$81j4Pc5&tH2+ z7KumCF1b8x@4Y-+aBa|j{##C58n$L%8g|R)`k%0Vg?--+`vv~YzczlnQ{J^pe%P0+ zJYJU)`k((zJjJzCq+w9rGq}N=fUn_o{SNY;$e*0a>KQX!YdjZUB`=89FO27Yr2cb9 zN|->uxPBp>9_>kS6qD&w@OJuFH;v}1mwK;p>@zSG(=Z(~kckRyjLHGy1AW>WsHTtX z*L%wN0P3{A>v04PXjH~EAzH)Va?Jj}iXY%0(VM z9~H-y|EE0z(OJLM!~9OyH#%OwNt&^LZVIr*+UvCw^~ckTP>d41eg00q`ae2~r7LvN zyHO{d)yr#-kPT<`zsZ-Qyt$d)qF<(Ap*iI!$LU$&= z(TR>yW6$iO^#h&cF&sE!uM+YE%BT6R^ZFHstw~`%g^_b#R_G5JkY|t;7nGM6S)Zf4 zub!e@oT`pYMrUuHcF!5~q7Uz^{|zv=7AgNrl+mTie{?KUw?r4ZQFm6^izDZi^~&DH zUgdw2^1ojBkFJyPaF##ka2^*hjHENtz^{yjkx?IBD%pRB`4QwejO-^o@%4E7T+haM zb`#J&j-M;k50-hCS#Lcv_r=%a>8?%26f`UMn~*`@pY(dXe9D{g183fhA58i~d@8%r z@%Bhf*S5%%$Bg@Zye8?D_~EIq#%q&ajaQ6&EnZptTKrJb@8VVIzmGR2y%=v={8GH@ zJFmxcQJ3^`yng;*{K(wFc*C;a#;5UbI%XgfvoITTFc%~H|K5|VUs3=5ivSI zHucT;QL;Vr&3Fgd$=pSD_q-Xe<3~M?tXdXvN_u`R=g2EO>dw2M*Jv7 z=I2$5lWA|nyHehWcaDD}-hr~BH{)lx>qQ?1aG>+eczN5K@w4=*b8p5E;T(MkwZ7@_ z@i*f&r{9cM<2<{Dp*P|e$YC@K&!m4v9sKOfdaI5X^)=VY`|Eq; zKV(C%{D*8}j^>EBWSjrpG#sDCpXrGDU}umAP7KG(kx8#ozC?3xXVbfkkJmCsb8lceH; zHG_F7reQh`C{xQZgI=}5w=L67rq4pOXYk<*<`@iFYoNYTQRw?|2qWwN+I)Yx@2~Ow zj zAJLvji}|wzOR)^iQ`DiRnOi`Q#y*a6vx1(D?xp(j7r!<3(Xql@19oM;HIK~40qzUP zXb<9%`3J5=V<27PWD)zrlYN(eUY+SXr~CHf%7A)xfASEbwtqEQdtN$6v3p8TigHw< z2K8t{)b?*ZCT*RO#>lAc-@z_w`*#gVo6=d-2N3POAFa`g)~82(04<3601|Vo!NV$P zGwKWIdfwjKWHZ|MKXPuP{Z|{VdL*U%$x^*|#AnoF^|}7%9(714t}Y zK4&S9vh^RB$GGMd{Q*y;g*4a3VLT=v9g{Hy8JLP`n2s69#4OCl9L&YwN&SB>Yt!2Y zXdZL_HTEDO7hoY4p<%#U5HdIS^00(lii_+2vgpy?lOyZ@_KgiIn7b#7n={1GY2qqb zaZdYB8|sjDY86=>wXMmKee%d8_pCCmpEGc|K)F0DW6s8!9meCV86DaGZ?<;!TT*mLlyLehvD@JTe~zC`1v8(Ixy#$WoM} z68+{;)R2AloG2UMKfMWU=)?hb<>&QX(4#d4hqT9!(@!8;Q-65ITYVhWMrDGk9nJS}$|!DeZq8MZ4#A^lAU23*D$2=lyU54QL$l{b)uD8ob*8 ze@5p2wwjMZkFG_1BQ``gwFu=Qlc!dloLRD=X2yB$Il5dy{?t$W)XI>!|M} zn!}LBJPwDKz4cvD-$^uw;qCUnx^7j0v|lU_z~Ov(0HX8f#=CC3Ve5_8fN69(Z%f`H0TGT|h2G`FwfW3g3k_k1At*OSGq0t-Ph0 zta1JScK+8Q{w&54EJbt{V4_$%P#&=Ad^Gn+{*SlU|MW^*>Q-;>|LFVbJAHqSZ^hf^ zFRHgi`#&z@XBJjq@F&u{@-RDkPGr{p?|NDP|I5y=VD6u&KZ=anLj{r99*oic!-e!B z6r%*uy2hp~eWLV>`wy4XE75(%yPfim+;kv1^K~Bk8q}i+2h7zgH~+7VKC=J+knu?R zG1MxHqB-<6+*BhmPyV++{%+|{oW>dSq7Q=)Cx?MZze^uK*}pA0oFj*D9v3i-sO_8d3;8ipk%j}q_V2|wdX;g9 zL+Tpi=@U?E9N_Rm*DAb2rFS`mXq~`+Ie)jryn6Rc#uQ}Wo#}rnb8Ei-4UCNc zt3!0ODVxzXPx~Kr1Iqt%%Ku^IzdF!J|HyIW|1sr1cNP9crGIiLRXZKk6ST{5m|lzA z&-!0a>3gV9-%$ol_ski{#H{G~+_d4Sd)mjnm8W;kaF4oeRKKm0|JTn_|DPT8HK_lS zO>@=%$rkQr^J5O?B09Hs9(nP6jrsHiScsAHIphZ&CDLA=G{|i?8LjbM-t zveWjYC70nay|(wQIz2C3 zk{o;7Cy>U?IE=>xRHRO_KaD-`(=XA~BM<&0xo53yGFAg6ru>l zC_yR8QHj@uMD1h|Z8| zBg>1WYkDXB7>?tRv|c6cR-adH4!_klsAWFkS`SVks{fuQ&!88580jw*zoPju1I%Y} z4r?3DC3-%2r2p?c^92lJ@I~|BUyz?875%T915ZZre;hd;6OfL{Xjo)ypUl8iOvAtpAJ3;8{UL(Z|jH;`*N)deuVD#>n|U z59qV+6Ss!_cT}SyPaCj6`cWNiro1e#zJmjMQg(yNXO3+#kWjjx9m=!2f3fsX8f`=@rpFMSwA7W!5#Mh zFBlbaM=M){cH0JX8rUDmSAWPE70%GBiqs9zOYcLizJtU14r=n$A27hK;p`>hEO`#i z^da&*n&=n%|1K~O!O`o)YRHR`X#$y7~G1C7x zNBUnNekyl5R*0YI5|2mD|2zKw(*MWZWPVIR2FlDWn@X0G2QZDkY4UI4)5$|=;%|!h zn=by6hnZ{1?Z(ueU2JS^_PDUEWcrYAJdcZ!F z`Yt1XmtN+4wWwVPl!Zf4o)zL@%hGGY);ZUOZQ0j_q7>(# zEf^KH&lwfwx_2IS&b}npo9`ey#qW)hT3kCG{eC?-BZ3KD|M)oh8CvRV%E?21j zpOF@d*_WUcTwLmaRNO!h0~~*ZHzm|`UCe@p&B()tUvIKTC#4D z^#@C>KhW3EfW{@}{i8YE`h)S-A4KmoUH=5yQ#_x2=a9S=-6&ghSvcc4x!*|&y<{H_ zE}>ep#K0hsKyYx^SFRvB>kK8id3Xw9Nsy9dOY(4q@$}*{#&E` z7vB%zzm)%J%71ZUO+wvYzt0rzGcXm?FdZ|{n)iRJQ~h6=cVs~Q zM7h^^PW`u6{a4+f#r3HDn8}}6n2kAT&J%x%w5jRQoV#f6`+4;F7&-qgLmwQXH9%b_ zlS7#_w!pQ8IB-mRfLx67Q_7xU`Gzuj3G-4MUaH?-c^s|%si9Z!rxJ%|sQ;syUW3CW z>iXJKiSP(=X>kV`+hVW_x--z zId8@Xe82WU-xKQK+SO4#yZ-_0Z2sq?>#F2XK;~A*LJ_(43H|e9Q$jJl1kqkSgO6%| z4{2vJm!lFjh|b_`>Qk?vM|;8@U7}7w&)t_Cx@Sn|3*^yJUacO{7TIq|4xNz<_l0BR zahyO84vBwJ9IR-Q4(i3dVf{CK;-3DYn)83^S6i%~Ngq#rxqc=6T`kOwWK)*DqlHnw zR9a{y+vZ38Qfd1A)6D%#3!UfFLUjIXq34{!X`I0UbCIJmpqC!4T|1OvUNn6G(U|~; zCn%%FDXWxQXW5;@5YFSB{r4A`hmq7TANbx|{e>y?RHR`XD$4(KzF(*I_X%a{8D%RD z(`%2ZN2mvl=WYVhF&R^kfvNaE?tdfxbQk&m%JTZf*8gemH)I=spf@cw{y?@wabmi8 zcKn%+8OTKQ0`-k~=1$YwOO@esjRVlDSN%qtAz2P9{v3EJVbr> zQ6GK{8TD1tV}N=i_t_sI$ynDq46l@Qe5m8i1wh5&cUcyrhbY;sKS3c z|EAhnSoc+;2K8t{8#>Xt#P}MHrs|(ZN4mTrMZS=x|BtNG{(t-Y8}8oTpO4-mUpU5( z<2ZpbZHpeVe46$LPSH=}45G6EqB8=j)jOhnyD#p~J9v%y?|5M>tw(+Fz1$99%^x0+ zZ<<>`KZhZlNB@^cg~{S%-xsaDW&XeQ{|$4K^ecHfQjvyn7>@~P&60+*g)`aSBTdW4 zJA0+^6Y5{b^#3&dANv3NmzKHm9&7v3`4gS}Gnvf2CNoSSGcXm?FdZ|HiCLJ9L2DJ} zkU81Ddszr`>CqgQ`Q(D=x^koIF?%m;P6`W|`|TmPh+K>%So1UC^E+$iOEN>BZ=A<& zDVE{DX=}KfGD8-<>Vmb|Lz!U(JsY(X)T_ol7;3nwMh?4%(g#BxnUCh02SWi_h^BgL z{pVPhjw0q_bkV!fflgG+e=t-oe$X6+2SXKEoz3qhp3!FgJ4)PBigHw<2K8t{8(QaE z|Gv=rcd~scf3x_TZO;a>o4M|U_3t=x#`fK2PSpKUn)pW?~j*V-DtG9_Az3|7QVtasQu%^hH>V|F8T1 zMEegeasN^*Ll#ybTKktx{{P=Ko{le7|1p|Kr;K`qaC~?qThJe$^}Ej^Kap zf2;hRr~E~<|4k2nPT@4ppgBYNtBmWVw=YxvE>-`a51?CHyi3{Kxj^~5Nd1FdwExXH zatH^wJx^Xh`Qo?gE>+6OVdkWN7yrilu2bq6Q9C>R5|LWiDL$~X_kdDcif(%SW|KrJF8X4`0H=Ue;Ow7V;%)wmD!+Z=jCWi&&LR4t;zdiq= zQlE^pS%bs$T14~r7jd^3ORyBnkcAa!T`K)+Cmhv=Y)1#ZGfmzwYDc(gZtHh;UMzh-4{9z7rJ%HE@h#`Zeu_5D}M2gm}~ z4lFU=t{*B_`?ahFwqL9Yr5~0Nq9ECg?0z zS3yOF`VS8I?r5)^YI-!c`0xVxlsqcYBaBkT4`FlG{pZm2lsC_$-(cJkZ?3SXuT-$w#@4_m-Fq_HUvOC{pckV1g6I2=PS5N>5xZiPpcMUM^_`G?*XTbX2dnLAMAoATZRo@?97jcx zb{*b6Ux8kO!-&>8BzlF-pW@#M?t75?fb?x&$J6xIbm1?2j!qH&XQlrOQN7-EdN=Ck z2>*G)9}Q@n?%Vj=Ot!eMEI%!r;m;alAtKUeNzz zok&fd{vVuU*H9wuKPp{4oE)0x$HNf2^BCDbWSMq>c>oufhtV}ny8s>Yq(Rgbnj=(f zZcwQ_2#u$tT{I&)qpS72_QeHhxQ9Q--Ph|rvWvN!OnOn8K`PRa_tb*dtklYe>tT6fLcu8E*+CG1%n^4-xN6&)6kzA!gO+^{y~n+-;)=tQrFI8F7xiQ z$&vZ@*7wb!ADAaUY%>3mUZwtXC`5d78N^@>yEX^fm7q^%$AOL&I47bE8q&Q?!h z->DCvkKF*y;v9x>9v3iJkrIZ7!U2EtNh{ zeM;Qy6-Q6MH9j@NGcqv?-FH|IOU}Vu%tNd6-nLNs)Sho&EPZOzccKg37#UwHQ2ym9 z|Md4YX8Fh2(l^<{Jf9y6&>)>IB%7q2+;#GLdb9L#D#|l%ohW`OW3!a4^`4<@?k2n1 z%>P>w3yZm3f~8o71L@M=1bYtBN9HeQn$t_qMydqsm$o|SZ;$V;(El_~JqB;@ z|2U-ng^}~u2h8FBnY=`~m!tnOfnt7_pcI4O5s%0|eFK$b|9#d7m>*CR*@;W_WD`c_ z`w5%ehbOAz8N+4QiDT$q>>JQ|+PmT6dZ6R3oj{p0#0$wDoWf}wFeYAJo*vH7tHvAu zMlZb&wN2ND!>CCy{*3{4XK@Zgc<1=rdFBfkMw9&V3)k_ZQ5gSB9fJ3ykd_Hm9I=@+AEu|I__f|Cjr}&fI!R|L+<7zxue_(NU=X7hUK^9Y5;%c?1n; zOf|Ov7thxl@!vT&gO8;LK@q+YJ)^@jf8-rAj1Nq4-dUk}+~d>3zBJ=iiR;6Y-1K+Q zzavhcwpL&H9DUQuKi%7G-+*Y3|LPaai4#^w%b9)eYS+69tlvi`yAG6%H|BfsdE-LZ zv}8&+kSY9UTp!kQUzH{N7jqMZgTCt8xxybc?4rF2cJ*Erb`FdW#YtC(9er1Xy~`$r zJu4=Jk_!{U?xgF&)`6*E%eXs3;nIw-8QU+63(pRX3q^T%Si3gWTmXIG+w`Fgy(g?L zxGm%s-4^ot?g(qr-y7DhcyFlo-1Xk$nfZ5y^%?JS?wR#)6Rr#!hpq~n(%(JuU7K9r zGXLtZweXs-ZP~S<=)$$(*$WfH_H6qzybuq&FE#!(-@E!xyT#L@vFbi(%^wr0lakdX z9OJ*my`J=eDK^98aW|Bh6Cv5rFdU48Vw<#)=I zR(WchYe$*4^P^FI+tee!LyLTOWIvVx`C*B^Uo!8p@5Xm=xAt{sEhdc#W%P-LwGNUS z2F8Yh{Yl~EE5;C2#dP;4@~`4yZ{VN2(@!vIjB@x}Nx^hh?Xof9Kgc&Q zihpB}hARH9Bj1f$df)!*jVt=j$Lz~6d>5nV)fK_-k^nXLQ`9tpARDKv}Yg-AU^>T91zjM;}o? zObv<8v-|!fA@NK62foJqNAm0Bm{H0bT!Ra^6LFupBzkHsSWzKa%YWu8Q3_TK&g6JVZA0@2|-jvO;6>QMYE| z`{eE9P26OXpCSK@Ttq(V+EUD6j*}lHpJKm&T<+aJL$;^g82dPXPhNFzY!Y)cC#*|4 zi_XxrWoII>MEfqw`0KKyu)240EO*JMkZ0_F4c4+-S85(lO;UKqc=Y;$QDKArhl28? z@L_&!A~*M$3wmx;*y5mxtv+xYH%0RX06woAfubF9V5n;P3cS-;~{{gCVrHYuM=$Hgk=T@@QSpXf&4(J?*N^T70& z*H0YMbgxD&T3qYGX*`b;u3wJUQfa}zY=NR9@jcg6O%uH-{wceoDe|b(;=@_>&+Eyw*Iq)+s5}U@s#i!`?8dju+K9}XP8%zo)Qi+?~jBuCz6@5o^QP`cJh%2=rdv`?|WaYr-FGM`G>n>C!g}1hsj&X z(d73g$9islPwZqh9=tZd&-Z4=PENcz)^o+pv6H#?yZ?UI*>|w7=H?Hc|H6H- zp2zQtISw-MpTftK_{8PndCv{Ex%c+iA$IQ(UMKf^-+kQQ_nz4IlbP?EDgN;9iW#w< z&psGCDKNV-jRl^&Bla(@x64yU38S{m8)HA@hohkr9w8C zvnzgYtfqWy_y^{y<2T0Qu7Ah%o5&xLy2TP1^o1x(x+b=d-L?Ep$KHiw!=8C#%?}wH zZeiZFU~=pT*@zaj<2LBBOZ*sHr@kllFXUJK>%YgQl(AvsGWqo4vEeD^^|Nn?ea`#m zProzvx8&bpCHt?F%b}|)@kePO$8!^l5Tt+Wh*pK<@CN(bE2RIc>^JtS(P8Zav9Rur zl#staB|HDr)?+mC3! zH+h#k;^E{~&begQV17rV`5h;N_C>1sABfHmZjqyQCTm4-7j4xZjALjCCzP<&XY4@Jugc$pXW~BO6xIY!s*MxI&80Z#-;yJ z8NHittBE9YJ=ui4OYV=IB2P#5Q8;|ZHw%k4=FZ4&z!}Ws@jvOUXT2*tth_%zcraFpd6g6xJlCAm^Dsqy;l{YKvitQ1;P<_#R z)Kn&itJ&2~A8mf&=ukhD9DWq{{|iQk#*ABIO_Rr%-#R9=pmp-4p>4|Opi4N>u1?gE zmLBWu6)&b;8oH72t(D67L-k1^CSSS^c@t8@n&Q;3wkS2M>q`y!Q>Mh;%YGIf!x!*P z6kv1Bm{2$;#r&_7uys*N*oGqdvx`&0_Hij;$Mop?Q^HPiS6WKgJ>Fb6W#1lhZ>sOF zObw;O%E|M}$t2&eY%J&Q0C~`T6=WszA+lyVF)d8r}Wy&L+ZgM!qsrS10E)>*g2 zHW%L>D?EN_c-Foj+p}(oZ9DEv-%~fob~5ieb6ac&S-fIY*zDf_#O~osgCP$456I(S zm?u#OQ^FFh=zyj~Vz+bZ8^*5;72{IFcbLD2llT!<{p{)R6Y^(x9xvcmcoC}liC6GD z`~j~+b0Lw82b6<|?RA^I2j4xh#9%E)NHamWP9#%gqm7 z9xBg06%HMHDpXC*Hs>eXxLI~MJU%?r+2 ztaI8EVbsmqS?>#p_usF6ce}KV@8YNUC0@d-cnzcO2#LSJH}MR1;t-DFU-1k48{R!N zBogyihMdJK!z#~RowZW?cV);cSsB*kuMBH*R)%$XD?@(4%J2-i{@lv2;rz-_FuXEs zJi9V%!sg&_Lemd+W z_c51_f7*T%(gL}kxqQac;XuaI;UFs9Q@Qx*aA@h%=0`kje#Fz-{!fR)%(Y~l`|77W z9ga+WIy8`tnZloJUhs|3ayl!t4zJK3logI%$O`ReR)h|EC;P5bS)uz(mPD~UQM$hcGgDT$I=frx#oY=|N|3bUt zgRz)2@osFzUeux;-^CAcq~fX2fFIL;il5_`Xnf?U(DaF?bc#CPEpJp*hTJUE@8jN^}XaiZR^q|eQcf4UKB}TKXdtM?cd&{aFDDxXN=;!@_E?& zShAYAhW%mJYsot9>znkiy1#*JWNwvG72DB{Zu}g-L_dCw*Dz_ib`WmC2eBTTP>XJS z7r(}(_v**N$MGe611qr>>#+$3QHAf~hxjRej$dOCuj2Q36Y=+JYoP@%VGzH^p5n3L zru+1%;BF*ptiRVGuu8*dHJQttciP;WUUg$~-MM(kKOYaz49CNIaszV#`;D$|A~$ni zs4spCwql#JTZ-mi=Krf3lRJ#F7H3`N|6k_++oy}Ug#8}Z_mcZ+>?77>Zhoh+NOC`O z`RU92|I7S;^Ip#xA3bk;bl5m9S%Ux>91fGU+*FX2%!kM-{_HPQpThwh#6FbbIh0}dJoQ2B z!Cve@F?M1XwvAJt#IuO%lbdHtZ*!$Lax3$ODbnLq>5<%&DZQphuW8aNxt=+feV*%U z$hFLg+0wiFSCOlkzab7hiPf;DMq)i$(2j58r}#O3k2evYAui+3@hN-;U%*yu$3grP zeu`h?O^li;e&cdniAlH#x8u+8L3|irz?blKti^h4!glP!UX&|KGo!9?2oFx7y ziT~!>vft?XCUP_Pg-uCei~F~d+n5cRhx_?`|INyvsUh(d+%!d5@`U{~w8iV$uQI;S zwL@jh{GuFtDXjX{C1Lftq6ePYwc~KA749g-FNJR9Ign@RM>B>dR*8LZVd&E zw}*|bw}nj|8P2A7JoN8#_EsyuwLhEq-SxPA@lwqZ(?(qNWo=D%mTQE5mA}CkmxaNv zkw5YL4UwJkI@kLrnu{CB8{*-3BtN80O%DDl9(p2~Yo1Lce`ZgxNUB($A^X2)J>!`-h z(7&Hwk0giKEg_Le{a|d(qspUm!b-SwJtgeMDYMeWiBGHJJWS8<&%dn9_^h!IV=#MW zxONSGxn(^sL5*G4~T4E7QCwf`=8`qHEcU;mkK`lVogEeS_AuY0@A=cQu=~WQ zu;;k>mM?P~g%3RnpTY0&-+imD@ZS9D`m$@~u06)SbJU0){m1^X%!SPARgpPLkAo+< z7Z$5FlRNaiCkNwo=1__w#Tmx1J}piDPTzZEFFYDrrI%aHA$vj@?YmkY;Lf)-y&jjw zt=W9kyN5ref8kXt9Y^kJ9%lcD=lka9e|GH_=cb8o33aOq@4?L~jm-Z2(17g!m3V!M zKN^q41KhhFrHR3B`8Ih;^o?JO!gH)K>nm_8_u6Uw8m#>jxtaQwo9NlAgyo)0*SuG; z{;gH+>x#brb?ke~&ppGwW9Th68>4?BJf4l*MbDkUUf8Yrg!|=NtCHRGE90Nh^M&C^ z{-%9cyx6n7SMdq;%Mx+_^X`=|@6BO<@%|$Ajlxiv?xaWmZdJ8;3A1kttA3~M^F81E z2Vr$rEc}N5(cay;+;rV%PRbQ;z3V#H_ACtQDHyin|6QMrhhpEnO5T`$t8qO%feq-7o_7O2V%`OFM`ZUxEF@=0 z=ltK3?U~Pe|MU>v#V&e&t#63pUwxPu>cr#X3~QV|G%@U%>$!_V_^N!bUzmTLe7ih& zMZ9`o;z)ex?wRC$CWUXj<9d{aqPX%TyC~d8;?H#N%%7;-S{u3jS}gop9=b>PNB<%U z|G^2$T=dtahpVqs_R_B-C*d{vP2?@O9fPg0a3^_pq$j+;uzQo6hBMxu-XzQ#kLlOE zpbvm-Tckg9vax;oQP(=T>EO0om~_$O*Xk4C*PoH2a49aw$1!-t6Y(;#@AZ5wS~Q>qZ=dgZM){9iaVgIj^_ryQur@suu1_R`*w56yE5GzmQ4NMWF0^1=bJyb(EK^Fk+~^L|97_eHDoJu z8~bPtd^_2}+&Nu*cYibwuU!~)jh8Tp-RJKO zCByfISLv^OOL<4$kL^2t8Xk&t|K_3S|KQJY)CL(JBtMKjPxOY5M)ur&oct6%NdGWC zicjG)_#D1~FX5~BI=+D?u@b9Mc(Xk_$o1HS&+zMW*h-J`r0wJ`{2s62O}t`otMB=L zj>Wm_f9xM)qsU8fIj%&RzNAm;D>z7YqYAt35qHSJ{V&9Jk^NV#jrDDQHP*MGB-X$A z_hc1wbF4pnG~St@Hh{VTmkABUW=u14|S=Yz_s9AlxG|W;!ef_&%|n-j zt*0*uTY8g*@b9E!Lo|TpC3?j^g6H z^lo|=>QRRV96@w8Mk89#j7oLH3VBc!c}TsnW~n|B=Gt_9Bq*itldqMz{+w}+@ts#BYh0^&}Gw!W_%BCU=40{?GSkdxq*B)sp^rqOx%7& zf7lPFYr~CF_f9d+4#Pr;yYWBpK|F?qSete#{4Kc!yRaYqkA5Y7knH==SK^0AXYRyn z$j{v`pPH+z`I!7+iFE(>(&CfS>PC6OKIyg6J9ikTId1OA56q$asd3{M^ck}Ic&)g7 zK>C+vH|qP{8>L-*KwsD;_^9{&SM9V*rSDDm$#0l%WZq7evAcSP^x?i|*uBa9oikl$ zK7umlW4`$ZuKhJK>C^D4`>uDr(1fNkety%ne-=J#Ip+_axBh?F`hT*TxrY7WQ`Y~ZuHO2896bTboZEMa_8_RJDljltN1-$!v;WtY0m19XFn^?-cIh2hrd1lDeDsbf95|; z*KW(yZX@^gSYvkD{3m&98M%K*{Rand5EZB7aToryj!XNfTAQc_hkMk2j>{W6-`ZpJ zh=*8v>dWUEa)$V!x9xslGA64S#A=HSg z)u^qs&Y?anlx7*HTXbnCJ9}w(Zmzi5cWKyvLEM}X%~2Q~+EkQV3)AdvWn7)?Y%|X8 zzB`3&7g>KM%@}o>`R{4r2-)O%Bl{NSX7XO{K7a+N!%=L)Ui<(#18FjVH06gqN|1+K ztnIV@XzDf6R$AD>&t2qBl*~yByUD%F(zF}WLSar?*zEdNa!bB3Npf3pnllm7!n4f3 z5I3&(ob~uVp2sEP={9si6M z@Fp(T&Y6I@*o=Wl7tcS1&){?TEq8aYe*hoAWX!`NJcX}d8@`3_Vn0sd=Qx6QOGl$| zmVOiY3i(G&qd$m8@mKgXzJ_e%VmHcAhZFcQ2Jj}vdatW78Sldk%)__k3I8rn$V<`B zgexy`4IhChm&nB~^mC{Ao*2*Vt+*e5iA8uEYq1l7AD82LOvMBE+tKQ0P%TNkIZB?D z%D?-r4EK^#+24hGaIFH+xcfSO!EQIkbC*xfC*S10lH4>!ot&Ei_P1d#K80)C|0Met=~v@9 zyo76U66F}g_4pww@ka#rt>GhvRt?K`i-TW7G{($wSP0idU6AE0sD=vZz4A@%?yQE znPE$IX4pz@V=iLT*7{j>wC$4=Q2a-`ONU#aAqhY_cNEX zKj8X7vV!}{>DH0EzlyA8u3>-J^;)uSsq@{lobR6Pe0Q>uxru%AxXjRk)@hlc4M)+A zj&aVNM;E%0aK3wvZ(rrxSEoG`a>qXu^3op)Ysj_C>!v>x@-rU_&yeexH?S{oeIvPv zd2`l7q0n9eTga`<+t?RPcqlxZ`cT-89Wx#Z#n?&Th25@|U=MvS_FZ@=l;Syh8TPwY zjsx_AsF?9ksKg<96{=mU!C`tW>Q>m_07vKzXmqU!&GZ(u&agiMj?&xF;aVrU=-o(M z@cmddCo`-@?t;vahc)!IShvFe$20Wx*x*_LHqtj?^9=tVBjx|xOPsT#jkHktPi|-4 zk)`}s?(QUaG4Ezy;`$zPFY~@8<$tI0pDgQ9{-3tyrT3B$t)V)|Tyfsomto~US;bt< zzQ*;#WG!=Dll4vRA6Z{M*ZQXU`bHO;?@zWcw`OT$Wt#&{wljCI?{vM3>}Ia^?`q10 zXN~YAE0`-63-6`Eo80eTm(TV;=K3GxKIYOC|I5EFBRfijXSwine>d66+?FT23xqe> z*z13=Z+5+ftYfY}?tk_8U*v{GGeZG3ViVTQni=x(4Avue(aeyCHCT&8(#((p^EJb2 z>|S_ZC>ao+&WTUt4(8(7_o;8+r%rI6`6%~=qQ&=xXP4d=wv(IPQ;03tif_wC8-Iz9n82PCzY?G_}PFDMs8gD zaAfbd{n|6Oc*Z7fe(L%z*Zzrpu`+rk^Lpk<{JFxreVgCEW*+NVW4NE>`c3@1lV9)E z2KYz3k6%x4yVms|yZ(FEx4NeaKjhEvxvkMKukwuV;%@EOuj6)pSB^^xALMQ|H+A}t z>yw>N8mykPvWFC4w$-$HwMIdt4}E_B}WTm9pM zq5E;YT%L76Satg^w|d|GA5nj|55R#>7{3prL&ay!uMAg(Lth*ps$PC~sQ$>+q2}>% z;c(r%LTyy;dZ#k)G|KzmPv6VGLGSe(KeM8+@qKsuzAxZ9?*0t5Q8@ViTKYc=iyvUG zd+%iaobPIJ{j2=BO89(+{GqU)!>@$*d%x$j;IG{OXYNU8W8`4f)H{`b+CS&Cf2<2! zgS9G9>rC>=&xnO*mRkR09pMIgfi)8wt&iA5Zq7E=M{Z%>N^VOx20%W`yq(z* z^Dc6?>m}qK=Dp;;Ve2Bu=a|dL{jQgj(K*Hk$qM&ZlK-du7wfY1p}T&B`v$Uc?v3() z_pyiYxA)dYmhVUGKuwuuzI|8{nqLiu-kx5*ogcA^QN&L&!DN(oE@}C zTg|BNydl)-$7-lCrw5(G=Io%`wJx;LThV^boF0^$i@kq_xJDj0p}%YLWnmw48Tp*~ z&o%6;FX#^=54&F3r@xH3iac~SDLg3dd|te$6-Qnb*EWkYSBpmn#E&iFU|byek~sBe z;!(HjFX3ixZesTc*&r^(`LPDy$ERG6yZ<$O*gc;U7iaS8x0vWT)$Um>F8_HH*IipJ zZT=L4_^oSI?tj&NFN&|<;r<1Djs4f%(~Yls-d6X%pZ~ki&+XT}%ccBUi{E;OYdtIG z-D|mj&9%||cmfabr-lE&bpP%Axsuz2b-Ou3w}w@Vwg0u1bIH7=+W+L*67BzT?f)9> z|I@eWf7Sj+0XAY2Huq`&V+*!o8;bBOwqr-B@el08F6>4L_FymeEi(SW|K~ij%rp0U zhjQ`&^Fgv=p*DKCe>H1tsM2rwzwVFn_WmaR<$pV0ja};*aRx`-+m4PlaSC1ZZv91n zEu3nEb*}LFb7As1d|kM1#g~NVOLz_M_kZ3eyw(c4k2C)myWKM=-h7PxTz<$R61#=* zKMD6A^Jk^7pTylSqW?<1%YV5|_}`1Gx%n_Y%3Uq)zFeZT`%Tt2ls2Inp=cu zj!|yY&0)2MG2jFMCq$W7<;)uR9#us+S)Bt&zR)}duMBQzJAzfdS2ARBU|d$OLn zj_fY+uh5BT?}BKJ^-;8;6$ht`3I|ZGQMw;xDWk%3D5dYi$bQEY%oiVLzBpOsy$+F; zZ8wJs*S{`|HVL2a3Y!LD|MTcS3%@&s%|8jdfAU`jF^Ri_?7k!%FBR^0yXTMOZ}11< ze~s{ejk_82OS$_E^B}hk{JqLQ`&;g2ifbG3ude+I^DX>*2_JUfm+)1rM%+8RN^bXz zPx0e&e%z0bv;P!6gKy(*c0a_u{Q4xmg4_Jpd$An1dDaK;mv{!xVuknH?0s7CEhJ`1 z|5%09$VDF3U@g|2xIN_0mi|xOuKq9mFOdH6Gxz5Dj?L5bxsh8kAzR{ACUgfN&n=oVdtB?Msh`hsftL;^C|0w^7c|1L&uJT==|SSR6{R zKPb1=WKEj!PqH>!{~uY;e1vRpy^(BUZYEnY_5YD=%ty&~*EN`*zdw%8 z;B$BqD^Z3P{1^|32Y-%FVKsh?QU1+Wakc+*Cq9QSpbE7Z<-hdf7XRrXtP@`MyzGCw zw?&@u5B}Av=zrnw@GRcrpZyQK+kbiw{s;EpEY8C+?!;tV!LMuZD87gtsKY)q;&=Fv zfAI-?7VB`C|8gTfCrm$u^|;Qr6=zHeJFJo4Wl#N`WC`gbEnzdet=8mk zL6LR&+ps-nf;Rpn=aF0=)}6M7`P}v92wxux=1mG47TyvzksB9G3ONh^AKKnMJgTeS z|9`U!Dg~_AALodcR=!*m+O#AjT%=fOODnc$u~*a{#TF~Jw52Up^i*e(%;bK*-!r+- zBr~}ub4^NM6VQNDmtZ2U8i*7nnCs2{T8mC>)`;Ynr)Ly<$E(h} zhAQ5F$fm{DsX2}JUx4=?c(AuZTN&3s;rNQU{yC=Oz-QD+`L&DqbyI%jv#{!AUmxN0 zBmKlF;O6?@#u^$CuKxgQ4=nmm>Z@G;o!9FL(kKa<;Wan{e}WI;TCR_4%=Hf;cn_9w z{oW7jx%R&c$KZ2Z?_Yqs;U0Jxo`?6~bKC>L(^qsq@_879|1{4oC;%H{IqZ{5lz5!= z5zg}sEyjH~zGySp!5Rcnc%hes5imi;n%d)ew=%79(-``HTvoM=nprX2kMKrz}KkHJP7~( ze(WORx0m?sCw`1Qaqg2IVgGj60pFsY{O&ch|3MXTVTzfpCQW)D`5V~sDfFfOSqMeM zv3Rjm0xem*i(ZwPUO5_S~uZ9b)bB*{m(ZJCAW@%}Bq4 zv15c0M24!EUxj>`a}Ds0ei7L*&ic2|HO2fa=)v9#eT2P&{g1+KC;&hBpc`tT8U9M# zFToX<$M$D{W=7i=Ar98Vt#B9I1NX!C;D@jmo`avlFCY!Fp%^M41TpX_cm!zbv|R?`y%^rZ{(JZ%T!5(;?zjGd{2N?`zrt1cqgrdb1{T5R;4^SN zEQQrzdzA6NyJu-R_M|ntA%uosRq7W6RJ-w4;=L9V~cTz{7sTY!57?v?nf!e4Zp+7$OcV+dSJ>6-_$ z)=0xf>NK0F%Op^?iKVWxfI7=u%D;=Lvn-+fJBvR4In-rvH{S<}eqVSWWhUs}PZ??t zWvRW?rFK)N+C>@ZQOZg?DKo92>=Z*8iZK7Z{uAd~hpua(WBPup^X&cBm$BaoYhVj} z4W1z_PmnKPSjZRw_zYYNOQ0@5y){BTmia)gUh1u0>aA|-v99~A#>V?C&mNvzdwGr( zQIE}IeA{W_J%t|oc#iF--mr&y^Iq!FkcYjP`fSMo>e2hCn=%fodfP_V|U<|zL_z*)PYlDna7ksy?qh&_J!2r z7gLX?E?iHV){(ZZh2+JenKUdtY$nZ3+M7Ip7VPb$zxg(c7r$*B(|v%ne~I!c zbYSo2S!LE%H)|~Rkp7{wGi%cX&NGe;%r}_^E;Ekp0^`^&G7p?IoC|l8&c7s$r{N+@ zeepi)lem8!?uYNfcKqDOzWvyfAq@(SlU7I{90c>sD?Uit9mai}w7bByH_bJ8o@)_m z&T{RsT^0LHtCDS77T@+Ox<-jr*C;9N8YQ#s=)FeH$Tdnqrm`=Med)-IbJr-7hd~x( zlRi0+yN&A~@^^CmLm~DeC_cdT52e`4pqzabP>HB%n9~!VXLenMk z0$Q-U!Na~*Xv6LW-$wERI<}Ga(6y7ifFA6jpaiBpvSAsM>^oR=v7Ln`((NM~OLWMa>P|KIbU#d4Z2lm9DN zM{y%-5FRG~N67yp10~(`IZG`Tr>SPuVEIeHgq< z`Trv2pO+Y8wVU!k3_g057VvCa4Y$C0xD~zu55P~s3HjiH0T_aJ$p4G*9CkZ%4Kknz zYQPQM@E-T+2k;Mgl56Z)_$i!%Gq8Z;Sx(7zGi-pn$&0H9?-THOD1l1of!`6vhwuxI z`!s$JbFC%de+Bz?LI5`7{w>^#fbKrq$GIM^g&QD=Yb_N<;dS^6{2jiFzeENYC7D4+ z46w2zog;iF@$eiaPtB17869^=IV06Yot=Hz2{ZGIGB;AEchgpInl^*I%mds{o569~ z5_U1}Zfc$iFVUv3k8yVwng4v4wgSdXlu_=eW?#+0d8$NKLESdSRzU3v>g>oSJM*N{ zmoi3%J3T_0jq=NUy`pd#7B3dvtvow)?R6Ll5*q5Pu;$5B%5z>>prz za6j|lkrB3Sj@hzj&F1+xTS-N;l^mEY2RPlcl>(_{vz0bITj`JinUIxb#`evo{6CxW z|7_)XXEXn2whDS@GyiAy%=j0^_Lc0Q|7#!pUyc7u{}+8$HRrBYEn`|-v8;cvg8nb! z(ZCq~M&i`OvCUJ=C)-Q^7kyVA`mkCTGvAC374L55n+53qnnnNDIrEJi`o9)3R{(lq z>HmU0=m-Bo`oACuAqaCnL|}k>VUW7Q<6Hw?$os_K)rItP9Yr7$@_?>x+Z0^K^|BNm zhVQ^r`27iFKrU>8H()#aws4)6K{o{9Rrmv3fWO0wgx3ikSjsVc`M;0yALKypLf(Ha@*fJI5Q?A}N}!Zy zPZ^X`9;kSU_aCx~aH^3tkMjQGIpy;5{zKMBc>f_A*=|BMpXT|uZ+5iJ+QU2Bv{|=- zH;d<=gZJNM-hT^ctCMS@YsqYxH8y*=9?W;AqTiQ3%kz&g0)!RhnhiO~f9w(0EIo(r z2Otf0z;~etD&Y~>!ZYQoa1Q$ia0K3h#khSMn&5HrvS`&6>NHnWvhj+Fk$DTQC>ILY z&YyLKzR4@J`COsgafNooD=M3FMWwh`Aj_d~@fG?!t}xgAikcQ)k&At`xYZLz-P|j* z^<3eazruLdEA;taVZ7EA+U>5W6~7*2FJbgRKgabUJNI1Sn?P68z59x~AhP+2!o+2e za0Vc-^a_0U|I`y}nZxkE*8lCa ze?tqn!2_+(243(%J9I!NbU`=t#D1vW%^&g{r2Y^70PjEe_xH2&xc{m94PghT94Dy^sF}_U+;vXV`uhzK`2K zpozZadCR${p_cyUQQo<|NH^PF=>9*%3w}p4^v~z=o`c(PZ-lk59zF*)?qNGb*MHl? z{eS$j9BG&3+|B)8HA^?(XBm6}ZiHLlSA_W+pr~(Sext39xG=59b_Ta&Y*YNV@p!i# zKwc*PRN-wYkOvleRvJXngp_d+qe0>6T3I0qeE zN1a?x|LgZ%xd#3053oP_9XhU2^WB%LNXL)C9%vz+-H`a#*}92y_2ahEq&Z|k5lq4w z;`Lp~hu7dv^5|tKE#m!V?^HZ_^j|Q+zGLtr_P3F_Qy)_vAOr0}5XK&X z0m>hP2mGx6*{ROWydM*IKQ7?;$UCriYZv1~yVSoJ#RCD^dFLiV5+uW)2-o=q@{Dj& z@soC`L+KYglyRYhK8z0L|I(*1MLCGFPcG-lgM9WU5!b@B0;`CAk>VoCK~Nen^AEaJ ze!5!~Y4o9#(U$_%*lV^h{^cO=f2dp9t@;thzvS`!ho&uEdV=s;&h!4i$ou~??|U+MT9%74sr$8XJP=DmYi->)uaj_OyeU^UjY5Kv>(GL#!OOC5x#c`#RX20WkQM~lHN}zD_aTO&T zXWf?Ls(`6?YOQlzkFUS%VC@ZuRk;thU)EZQOfpFFp3omT_Wy#@eS&i12_=IA(svzK z223T_TIV@F8+#69a(otCz-`taxUXv8(#Mc}3%DfV7ceoO>k0X7+9S(*-eL>^^#VA7 z`$>rOrdZK&0}BhR+I_UYK+|c~HT@jVP1-&hc@TN1khGqirxxBb?pdFZZy#-w9NX4A z^PT04rSp}(Vm@OW<}=oNzH(S2EIXF<^fu2|CM0c~uSBFBHyiepnE7(z*TFH#P{G=J z<@@KW?7)1=I`dV-I(*g0s=f0W>ph=uBF<-SzG2EE4T>7V#dN?;8% z#zQ-{Uae$E$zyyq>y4!q4blcOs0{m{(pf7e>%i4q`}363yFl6Yu=4m=c5Y{wXIqt( zpI2oSdBfBJ!YVmFsA4D^8B{4$bkOK+P{Sh^}JE`+T)Pj2}ZXWF33lX(tMbv&SBHzV`I+qTrqbj2AZR0!xBIqJd@G}OK$#2kt*oK}Wyf?erlZ=*JvYGmAHB-ol5Z7U zsIdx<52y(L#nS_f|I4>ZcjsGWFXdb1fdQU>tpCQCkE*Iq=pt!egS~bM>wj$hgzCI= z>7$vehL}&NF@g0z=Dfuid)EJ$%laSSVJt~2v|;yx&&wG7M&^IeUe#H2OkGvS)a`ha zdfzeXeaF-{^_Kd>Ab9C5g&>SQ0t11!GzhkBCuE1jo!rNejNP&S4LM(WLn+AA z?QbaU!8eq?{{-#7CusjYLHq9s+J8?l{_zdQKfXa9*&E6qe?tXOIQ52#pcs1zlwNv+ z_TM*X|9wLhBX6j3`h=>^&5Wg~8At!gDW3nQR0s8^PpM(#lo}78QWLWI@+sOsPB9+w z6!*v}+CNUI4L@G|`k?*5DRs;}Mfv}fx)z^OH}qid?R}fMv%gY5A7S?M(d7UiSPt@W z61fa==l@c9v+0-k4&M#GnsOj+v*AH-LmONR)moyO z$758<{O_tyT(8^cPx%JVr^W2M6Z?;l+mP{u5sN(@HpAC(y90R_Y=e5Jg70!HpIEg$ z0#{MCGM8jFaQoT#%!;jZ9`7{hhx-ZR5tzqw@>8$~I>#7)GTYJzUu6u``?OybTKX*R z%?nr~@hf3{@D` zkitEZx172k_EbCTJy)?79Aup}_y5((IZs;%_fpI4Z z6+;=6LIspV74?QnsDWy59iPS8xYYfi4jOlNso^5^iVHJspzg_;Hqch2X#@4f&a{EH zBTXCVz$(@_r#-WqvRxPSQr7E%{+DP21ylC(!*cWvvY)kaRrZr#Y8BfRFaMHt2+;dY z+V$2@uc@7-4@5h8Ed5;90RuQQNTZwa8(JuR@EO|AZ{_{}drNC^%VO>N{2tQrC!~Ke zZTO^V@n6Z07E2r0zZpJYLgC%W+u>fw!`}yO2sad6j--B`JYJPI{OpYiDTjmv`=ne-%!c44nFSxP4$04v{@v)mmpZY7di?LHD>X1yE)P4CTZ`Z4dxW*s*6S^$RvLNN z9$|gZU-G=GB!1}7v4!+qPT7L<_afWRlBdWHWbp>BIpSZFhTE*`^!k}<>+n0(R`OGX zYp%!HOH}+0cGA0^dHcx^&_B|H`<+~;cV4I7T>4aqmsumrMLdn{{VRPV*qgu|xk1Il zoVO7Fzwqsv0@j(#=bMBUaaaM@fQ|bz z4*AyAl<|2s{f7JhEPRlYZiU~ zP|tXUhB-Hgp@FvLX5W(QRcvoo$%00GmU7tFabsC1TM{^-2&(5cs%BQBDq|Z}g}?Hp zjjEXCR_O}2%5W>iPXm0K;{(VroCKy#+I|avhkwFs`cFR&SHl9h7XAbJ<~FMzdiFJ_ z7rM4PtnU2|t7C5iZ5a-0F@A1!l`T?`cxb?k45B8kD!9Y3BWhEL_^8qJtjVf|{rY zcvO0h_up2J$^%ZTf^aGWO{%)sr0NSzv;#M(c43oTbJMK4ZB4X)rdti1udz4XYTDRD zJFrJBJDrxB_<0`ns~I%9&TI6j!ZMJ8PBZ&au+Z)zbczX=S==m9;3-$}X!_&Z;)% z7TT=5^R={pw=(}X(<(e~w~8ifRgC|Vv$eE;XIf>uGOcoCMWmMT^mXEuW2@fXrkZ`V ztQSWc0@QK7O5`2OX&2#~O`K1tySBOI$eXCZ{wS4KVD1!Ppvw4 zw5bc(O_)79-{jkyZR(qAv-)xKuc}pGH*w*Z(2iO$M8Xz1kYf#OtkvMU@qoaI#$vN^%4mQ}~)c6jAEV7FQiy3}^SC2xXDKEi3A<>LD?F8V*St**H)bsx^M zdN^P2#Vo52fBoZL`4`)*0Pzbh@z58YWrerftjIo>2JknymH1prw5q&ys=n-2&75qj zG_8*LTG^D>>zJ?PQ(Rmb9E|-$+mJY)luiEQ$Fqm_B#3l-(o!!pR&smt(>KG zN`eQK6r!L~z?Wp5>MRnAPe9Q;0Q#<}z$Yb~EI(bHX zYQ?SLLbla7oozKCn>lao_9V--ua17FI@NC_J_~a!`>Y%*5t($lp8AhVjwSW_20EuK ztY_Rnz0wx8D;@bw+#ZE2+_UG_GycDxYnl9CP_O*O^(t6YPkOmD&2?KmbwVWx^(w`0 z*@bqMBP&MQN$(R{u!8H~on+PER=bV-C!9LMsgJE!1F~^(rq#5wUd>fGR?EhE@;}G& zaK6?+yV|xTSzhAj+ulN5ImhaFDbecOQm?K9_3GYV&)PXTR_|nv)ragSUjAJ!13W4BaO=*p7N_Vv> zBd?9|-)+hYBw5*)+LW`|X5~(`DQ~yU%Ew>9PVQ%Srd4#gO~qc`s}I_&($iU1*>s!A z@mF!YO_jTBR@FY6RgJ9iwy_4XPp+&s)fLsM-rlALXym!mglvwq(eKwr`*$1TquSJ3 z)TXuzZSoSnkMp&kZKHk9NBbA||F*Z)jqGu?sdr1Q`jGwmx&QaPt-wev?Sq+Ch;YIi z+Z5T~rU7&t+|ov2(<^&gqLs+KlC;2XC8v4O-%dN2S1H_6sZ&{2T9#Moyi+n(*saX0 zOe?F?t8Dz`a9`%`uv>Xs>{dRqpvtSl$#&+yasP8~mF)2H{PU`ed$b%`vE8f6onF@e z@v{DbS2bI`wEubKI_zcqgO@shR}K5x)p(KnfAN%>ku8&6xno`OAY12f|8Ja1H4lQ zFWRlrJRf7Z?W|MnQ^kTrt1`_;9VOAK4)|0vmuI#6q`G2gpSqOII@df8F8Nf{>r-*3 zk1?P=P25ad`Ub`qkDnxcd^{h0$~su5Y-A4it8JT4c8*Kj=~L2!JP%@h`tU}|SF3z_ zEtYc3L7xJLeF{$b6guuxn0$%w+!$DVlKA;}zLURs-0#TlEFWXUed^8g(ZAtSzr#oR zBwFr+iIxZ1y2Qu(f$QTm&w~-32a`VflCmw=b|332Q!YB2XfN|B>bAPSNMZ{XeT+Ror{k$eQEa|J2uAjokk` zbFBL5b~WrI{};8>7uK%krR{2oZI^qMjd4`%YV{^rZL8YJ|0K(|xScf$>eaC^$LieM zuCC1u%qL8;diG{py*t`j|Ds*}yW1)MBw2y!BrAvvZEh!hy^37s{y$sK`WNkVBzPGA z)`YoCe0frdk|BCtG#M`bS$RpVPLtV7_jn%-p}0_V3NK4dUk6)ynlxo8S`K29ZAev^Ne|9S3OJ zgRZ%>^+C^K+V-Fid;h{#`8U!wczM7IUL3GO*u#9lHnM6y*FSBAgkxLPNcopGPU?I~ ze26P~qfZWSE(};H3uzObb*s{jhpqI(ZpuG{R_0RLMyJxO>=@cgSJ0-ph54_6u$8|p zU=?f~vqLTAqaq==0>8oBJ650zPPmj)kl-G1o)?KWj`J zAGCT74_dv9i|H%!Tm48s#|M_MHpRh4-v8)NA6H~?*c#y87=*&KU#W<4d+}7dRkF*Y zT*$Lio<|n+QeN-$&}Qkds?T{;6XUc>FL_kfOL-pm3cE+idpvUB*Li^Q(tZye$6Qal zs81l1Uh*iDYb0xqhxO8b#rx93`_#jHY7gy+9-bE-_2hZf+evvH+3)qBPm4OM(8tlF zZrsAt9z`gN4RGE;>J$Dlj{>*{(*&ebSFAZ3U_S=+XDv!L&>{HB{d(gvJddlx#v|EQWZC3i_ znYnYRfsmE9Wxy)lh{>kUJ;C%G;b_Rqvr(2bBxSM<|cUuqqa1SfvZd z$GI6+8mIag&-V)CBWq|kLss8J9|MwRO4}J|;5q#}J|6EgRlqk;CX1`zHgot$GX#N)EF31UFRD59B7la+4mBBhqQZ@bNVloC;-QJ$GrwUWrga6zUdP6Lx-b~{`y|4tFhPW zuIja%E7nlYpx=4b8aZas7IkiolE8L;jqJE*(2i7jnzl#wmG7f1vSDf4qy>g()xU0zE2_ELQld&9<+%2>^q9?sW1xm2Yamoh(MrOJy| zs=~cem07W>+Vz&I1F@=s+_aU{H&&w8O4hqx%J_<-aO5RU| z3z_FuDhslgE>+HD3ccr7%Kp+)o<~b*-(RZdh=U`rQcjLd*|C&w5z(Jl3(q>N!(TzR zBwB~t68Xj!@>6Vo!e$-*Ym)W)UvXQ`_Gh>cU$9f3M!rsY@hfxXxO1MIFURS}gq;uH zN4Few{5kfgu=|jGa1EqBOP^8??P%EVK2N(lV|4T2cWjq%%oW(<%r!til~OtBChS@kqQ%F2}3XU7)hV@$>`7tKxLLDv?!t zomTZh+6WlOP$2ctioLrtiPAhmRL7`b4R`_fJ?~?@HC$pLVMf%OM z^Zlz7D{)c0k_b1MK4!;sGWTV%Qg+7kzM7-7z47R8w=zyATbZ#bR@T8}D|@5e%0cFy zPEa0U=G*gCfGoV6K>t^QiaAHg))cFhxRgP;JAvyvf%dNiRY5iOn$CRH?o5!YihkUp z6ssP4!)cq$F?#o$>0Jp3VK4Y<0!t zQU6b|dJe{`cVnm3cQ{`Cq_uyVe(%M}R`8tN3Y|@{!sp`^>5Zqp%{a)p$yVOQ6f1WD zV>7m|SAJ}figvD7;T*dBkD;v~kM>^oI@KVnFXqY4aklM= zR#Jd8WE`ktN3xZSUMam@mXm#H2i7Zf-+J0kQmk~w8)O|I4foSluyMVzcdb|FqwCZ$ zo@90JTSpys9nZ@3^j}s}cT2JQ=6s2{Ek*J}sByi5jfqxd3py~aVla?w4eZRP&wHI} z_pDR>BxOjBZ``|14bZ$~otnTMvraA0x+0Hf_d0pck%nxygO7YpWWZ21_dzj~oBM(L zH6^fI6}(4$Z1+LN!ew&cp0MrrTeXSj$0jAlZc-8?@7P2hZjuvHAa&m+r9t}s zP5K)CzX?A0G5i#Amu^yC%qHcp*u)&cO)5kdv0aQTIlM`w7dP>q*`#vp6{k0G|8G(i z_G)BJWRq$i-6YqZO{zPvN%hcxy%CyNhp2hUt@MB1O8w_nc{bjv*3Mkodn)B++qd{u z{haf6Y^he~;YxKK&sF!qO4>gv)jOR_|7Rt0i!0?HuT%hgaH^8}Wu?NMmGoOwYQSBo z!Ny9)0<2bl?`rj2T&-U3YU)uj>UOVIM{Er9N@MgK=Ws$IltVBkMxli<^4nK4?kI+J z3RkQ7+-jwBeKgLBQQf80lpj~C55KOF)$*0CR{Pc%>ZLLALJ;@XEir1@6(jeG7}9Aq z?L@0pxPLY2wwiX-7|L&}mD?GkysQ|#g8#SR66A1>a&L?(@?sc6xmsoJ7&$pMWzT9Q z+heG=#VGN>Y9;OBVShYE_N}Xxy?8bK^fAhuj8PU85=O?s7^P0f(5}P9I9*BoubTSb zAE^ITQ~#?boP0TvDZTmB|NcPzFQ2~8FEjq>ugaYIva&|L%=4f1kH~{s&6ev<#_@cW z@ykrMYPj#SYGh53rbn(VLQL9EerH z!Lur?dXMr+Eaj6}#>Dy=mjDzp| zqw43fwqOEl3~pu|JhC~4aXX9V$-VGBd3yh@R_tw;{v~hK-x>SyFWP_SsblK5tp71j zT?^)^8`-mD9%EwXsqgf^)W78S@)J&g@Pfz?;fK)Sd&K)$c)`Sf`!&jYko7wsW&KWM zA=^c>sJ|Xy{Z1&|#`>L5zJv8U9~@F8`>PJVuWHs#r~%XQZ->NfpH$LT)&fB~@(#+0 zJ!KzjebCvKHixx7TnCjgi?u+oXDy&!(|b@kS$w}GZJs)z%R&6@#2xWWTuV2Obp&R*rfQEL!fYmzB4V@Bcvo6f!QLX#Z@+|1$m?S-O2T zYva#WdF*UeFh;v_ayIR2>zV&IoA#gCs%4)Ge|6_syWlKq7+hv817s81&BzwU7`mAo z;bAUB>ni4lZe)Ha_^`J>_?kK%WPT@&(p|`I-oHKjncq3}NA+Rv-_88ci+^PPAM-=; zAA&H)MqmK@AlRm6Q_q}Dy<@hLAenNJ1Dx1XAeDV-kj|LA493`HBC}!``_TBBa+ae1 zg7dWhqW>QBhr$Es{}ST>_tWRQmvMg&&QaN;i~~egR59+)!MMLH#sMO04p7fZqkkLf zsB_gH7-CEpV<55^`+#hwzrAJ8LAkMeaBJN$q&EC{!N)P}&~fQybzXc;T^ASw0X@{` z%vutC&=3Aj`sg7DAqeNu{s99p2nQ)=|2OwRCJb?J48lR~@j|v+fUZ8ipG_YijKS;h zCj1`WhuPFW=ficd299#N&Nbe z|A4pP1n$S+2-HzGXo4%a{|>r&Hf_3*@L4NAYj9ByRl4%&b=|4P}+d-hy~H?(6|c}I&OCqI8(1Hp^(@y8 zxEJv*0y9R#2i}W}qk$JmgCwwcre`33OS+XHKhAaq@?Ex@k=-x^^Kq{yEf*qx%l3aD z|AhP-ybD)RZoC#g2Va02jU7MNAn$;?;XT5982NeJHz8ky71&q9&#>n}3j7Iw1;~1+ zfmV2ebBB<(;CB?c5qS#vPxvzSYiRqp7Vg0QRk#KFZSWVux&!$);FKZ z`!sV;83#1N`X85B{{vaZb~&=*H0ysLtJtna)j^lC+Z6WLEXk79oq`*&a$e2BlRZY15X{{Wta{qXSw#-zc2!Zqx_ z7Jh;Ijm8aoGWImch6>z1hg&nU6$)@Gg(2)W;9iCO1ol&~jO{wM|Azew*gwv_{vq~j zkZrhs0eK_tYt1tN?txp_?qgpF)?$Ad`55ja$Y+p0gG1PlKneCD-V=pTa)2@GRa3mn z7}Fm6Dt(!!=+j`VdLv`jp@#W?)o#YJuR5jDEvy5ymAQT3n8Uh1kTQ#PfROgxyi>Nn zN?DS1fFO%^O=c`($O zysLP(g<;T3oo`98omjTpm(IL1-MF`~ z-HPUx43o+>OZF;10M2?uKu}Cin*24-dod@%v{u4|fyxi@5y^lHg8-?;x5aQ`E7UgG{o=CPfRELg(*k1S%l7+JEJ`+w)JRW^(J zA1a^{s-PNbwoIw^GWS1z>o}$!*+3YL$R@U%ku8MbMtaz8MYgfui}bPGj_i1e`A^6$ zw!4u%vl#zNT>FS~Klsrh072{_@36v)nEyl=1IR&+i5zE5co^I_qJjM*3Sjp`2!haC z#G3NZ4}H)H9ncM3VAi5Y)y*rmmq(NLl*$h^mog zY?mS{#wl|ni^?dI=TR0v$C~=cymKSUogPs^8fEt050!P6Ir)$S*^oBEoPEfEba0$y zPXF9LD+TEUJM#j}S_w&)nDb9L|1Vqv^SFkt;(Go#e3$FvT4Vvwx=-Q02p+@j3AhdS z&$3;DtblG9gyp#X_&u&O?4xiDe#Z8%k*nBuGu*h5`wMP^wQS#kdX11TfehRyr`|&Tx+i|-Wp2z(~_#A7=>|pz6*dIb}V*A_3`;ouE?J?vt z@S@p97LRCPhcmssD$ae?tH=>fQ|4a*aWx2X81DP z1}}pt1M|&14Z$!FRlN$#snZyY!y$MLUI*0C1Tb$)M`04)0Or2wO&}Tc7MO4UF+^BX za1u_zFM(vxui)2k2BzUR@LPBX&H~>Q*Sqk0I0t_KzAdiz;XM2i{sbSu1^6@k1^x;b z!L*pfB!DYAtAA$|+kO)bT3=VKY3Zz0BqyzK(l?hpp4LOhtd5{kU zPzXg(3?)zsWl#f$NHx5UA57yX^>89)I%GhBA-cC9y0;*@ zw-DFDsB2-=wJ_@1jB5*Y>+@#&FlU(}jHyh+R32uQA!^IUmL0X_V9SZxa=OTtMO?(8h^%{@nt-1f!pB@xD&R*U2r$>QGqXRgRj9oa4&3!ufsRs zKG*?#c;Jio!vpXjJOmHJBd`;`1-sz;@EAM}yWt1$1pE;8z>nZbcnbEy)9?&D3;WS$SQ8OuOI-;gCYNkZZw5XX8H8Z1TcGS#?nz>OkFKXsv zzT-DxzT-Ecz2i5>z7wc2X3!k_PO#9Jp?YJ6?NQSld)Cn!HND16?=fbU*?Tt2_&=MQ zWz0Ojefq_-6&1#;tTCpy3G;U)jmE4@jhg0+zpHF9X06AV9VWDQ?I!$p(*nkhVsBFH&5B*a-w0B|n3buRQ4jbq z`(vX!Vxv1^qdVdhOAkV9Ty#fVbVpot$6Cd%Wyjj+j5)&F6uij znj~>}jV4E2G$G>FDlVD`anVGGi&tDU0pg+w5SO61sPb{^6c<%FZoT57ipFhFTvWZd zjf#sZ6-UGf4U?my=Qq)do@K4p-Xzj{?Hc}O*VYZZ@Ug5uXHe!b#1D1M{jqX9%UiZ_Qw zwMtOJO(I_t)+mAf>!RicMHNjjzM?t3PU~(GSshh-opFxl_IjXr!V^vEJ;A z=KBV1xJhJuG+{Ov=V;z<)W(}c=0}rhqj8Sr|CjWoFKwpeax0~l&E~(EqRVZ%ZK-a% zO<&O#ZFx{zev?|XSGVi-eY!(;+^IXBxmkDM_T&?~Q+Iw%cYaHE{#194L|{N$b(iki z@`&!%-M8!Rul`hb-?vM5AJA9z)%*0-hxFAagyS}CGk^G+wmqnA4{6&Y`kL<1J-6!~ z%O7{Z#z7s`@X6B9@r`l*r9Lgo6qZh-G8U<->& zyB^d-dT1N}C>T-QAJ)Tn=;3ea;qU7aJ+h7eHtp26^lg1dkLthlU42ixwCm=*H*41| zkHPcWb!b4}*Y|flr^oczEziJyJ+|v9J;vtadTcNMI9RL4_4w_2e3u@7N{{d59|ySp zRF9LGyS4iv?SAeF{Xjo3|2^=OjKJKhC-ekC{!l+ew;%4(9_=xI_>uNJtvx^1o}cJP zdQwkr(Ua)+B;KBUNKZbjCrwnyzb7&G@{fbHdh!K5NlHGYr+%cT_Ufsp_0%&@=&7IR zDbjzh@P((nTeSBf?R{8#zpuSdXz!k1XYbOpY(K1LAJenXYoDIebGsflM8o<>#_4%I z|D2wGK`&^(eyktw;U8s>Uer(YlRfFJPqY*s6a!78Ynl#j`p%x9fHRRDytA^S%C{MX^l7ADBYuqpG!oQENF!nKsN&{&HCwOd>eX7k>e8#NdbM4z zc4*Y5QM*PHHJYW-Y>nnefkq28TBOlpjh1M%RHJ1YE!Sv;Mk_U1t_iDUP5jbSPbiGIS_Yhq81iTZeLVC|8H_bSPhkigc({ zhst!QLWe4Ks7i-wbjYPcbvjh9Lk&9As6$OU)T~1-I^@6*yYM7Ab!G?A-`JWb?l0uP0nDAGi+CQ39>s);g9lxw0w z6P22%(nPf;YBW)+37017HPN7nMolznqFECy;;B2~(L}2z+BD(S1OxxuHPNAoPEB-a zqFWO^n&{O;pC;nze!6G2UcG!f>%1@3}};9+t|N&$ zlB6TaI^xg~r;eoPNUDyc=}5YcWavnyj%4XbwvJTlNR^J-bkwe+NjjRMqnSFIrK8z8 zTBf5NI@+nDT{_yWqdhv>tD}87+OMO29S!JcP)9>L8rIQ>jt=PPpeAjav}-a^lS!IP z)}%v|PEDq0GF6l5n#|B-rY5sAnXSoOP3CE`K$At9EY@U+CQHSjl*uYhR%^0GlYN@( z*Bg0yqg8LT>y1twv+G!*j^*fBu8!sDSiX)G=vc9iF-){f$I5lALdPm~tV+kKb*x6m z>U6AL#~O63QOBBetXaq0I_A-_Rvl~8F|UsKbgW&+I&_S{yL7Bu$9i-{Kg1loT$`^JpN`Q@2x?d$mef13iumY$lu7K zw*tn%R&0z?W0bwsV+?HN#;D*Ql_s1@{$`^}C#sFCHHOO=bvjXRWW$Vs-zH;gHbx77 zBi$bvC+t!Ci9|%yZfqX@W`7sDyzSE4b$YvAZ#U@eM!ns{|LtaDv>C%|44*MNjnRcM z=; z*J?s*HKDbd&{~bxR^zqRcx^R@mve-i@6MV>iu?h^UdT zDLa3H$M71%7d1G?l>H;aglp#v7}yf|8=r|&$)qjd_pMXlzOR-)=Sr zr%5tClgw$7xKuGX0^@&#@oo-GG6%9hndcxzy)hb%K{&}K9E^{IIh3vF0VeR|=rtG> z1Z1{MQ09ynQO_S4QO`4xH4nb%b!9R}X1~Ln$=n3SFg}eS48+Xt=n12GW6bz7*FGK) zZ@14_Ojt8VnLFCpXS|r(ZANJ{pRp6ysLiPC(vtoov!cw@!L> zvRx-Tb+TJ0gE|?~$pM|R>6Bfk5_QTU-j2K*Po;1|UXQ2JbSguqGR0exEBsV}&NLLA zX$T{VXGF=2sGJe58BsSQ+<0r9v9`~MPF#9)rpa}ti7=Y#W<+~bG-v2cbEeL;6rX8f zXG_V9sGJe4Goouo_@koDb*2rUZFN!MP1PAMVR#9{SEw`X>A-e7+nwy|wm}LQJNvrX z*Nf~m5x}67_c2{Fhj#zlq1!l2 z+nO|;R4H2d!48R#1j*n4C!|0sq(M4lKqh2CHsnAq7d`Aa#0_SmbLl5+7+KJvy^md}J6MdZM<3t}P@pp1QC*e5>%Sl*H!f+CX zlQ5iw;Uo+vVWbe>6pl|J&MBl%N&yh}6ylviTBOtf$E9#w8|mAI-!{^<4gYQUZ{wV8 zA&6+&i~e5n#Y>*_kT*TWKv+GbYY$=dpmQ&N`-pQN@%7{1kAFXY{T$;*A3yv2oX1bR z{G7{AKKePIpM3OlPCxnR=e&OM(N8}5O*q8CPrUu;?k6w(oY&8J{iMGiJ^bXYzXFJd zpLqC5A3u8e$rnHAq;x2hcx&{sHt4pnm}U1Lz+>{{Z?2&_96w0rU@`e*pah=pR7;0Qv{eKY;!L z^beqa0R02#A3*;A`UlWIfc^pW51@Yl{R8M9K>q;x2hcx&{sHt4pnm}U1Lz+>{{Z?2 z&_96w0rU@`e*pbinshpV{sHt4pnm}USw?R{z3E) zqJI$mgXkYb{~-DY(Lad(LG%xze-QnH=pRJ?Ao>TnCoA4bnG zdWO+6jGkfi45MclJ;Uf3M$a&MhS8H(%XAn$!{`}C&oFw1(KC#mVe|~6XBa)h=ov=8 zF#3hjFN}U+^b4b37`?*i6-KWxdWF#|j9y{%3ZqvTy~5}fMz1h>h0!aFUSaeKqgNQc z!sr!7uP}Or(JPEzVe|^4R~Ws*MlbR;OumN6*D(1SCSSwkYXp5G=o>-b2>M3QH-f$q z^o*cq1U)0@89~nodPdMMf_@S7i=bZw{UYcWAs-{;V+0)|=odk^2>BL4zXM0PkEM&IBmXZ|_krC;)@5lFbLga)wNc>B^odg8=wIq?uX?2&i9?~JKuLcf06&opX2}f&h(wFl z`~GC#pX~dSeSfm=PwuNH_tg`_6T=h36T=h36T_4H>dAfe#PY=Q#PY=Q#PY=Q#PY=Q z3du|2Upu|2Upu|2Upu|2Upu|2Upu|2Upu|2Upu|2Up zu|2u(p7@^JcTbE@j2@)?*AwTH`|XMKiS?QBnemzNneUnJneUnHnd_P5ndO<~+5SJ< z|7ZLEZ2zC_|FeC4wy)3j_1S(t+s|kF`D{O*?dP-oe72v@_VL;NJlmgV`}1snp6$=G z{du-8&wqFR`EA;ov*xWE|NZ;V|9b1!JNN(mcJH(Oe}29-c+&Z2*q8fL|BpZQ{|wvA z@T#?DU3UH%nXo3Usm?#6gVq;o#2T~4Enhe4>qhsjL+i|c|Nis8-um^<jY;tVd+O;k$mmIsYzFF7Son`08ez*SU{L}r-^7UO`-~DQhTDH+W zw|srq*LQt=&o+BImhJZTtW(QA_FS&#a`RY=F0<6@{BvkKhr^wJ&Nlw9Yq@r<_dA^* zzYSP})~GdRb*-K?ZcSK|)|54E%~-S6e1H9)AHOa1|25?CpSS)T|M#ue-o}64y8O>) z-^S+t{n*>W(tqAsKmYeFC+q+7*4n?H?Qeb2TI&25ur~wtWWZhs)qV`vg8`o#ShiNJ zb!*ewwsx%p>&QB>zFF7St#xNTSdZ2p)}Pj2KL&sY{kM!QCH2Ge3s@{xReR>%VjMV`$;u&;A&?{V}xaqrX1mGH2e}v-bPz$55|- zi9!3*f5&Cr_Jj6x@UXvr44(L)x9s=eaew`An+#t4yfXKU_g(Fv>$9i6w|@+t_Ser% zqV(T#jc$O!{Kd7U({#L`|IZi`d8~MyVR#%^w*D}ga5vu zcmDm@h28(Tz5ey|Km228=jZi~{=)wHkbVC5`Yv7U^4B$8{^pSN58 zuKe}ZsJHsc{`b<8_1t0Yum9|2|85@|w1%uN)>muT8tJc}*Vy0fe}zNt`k}F(w{D*> z)cr9u>7Sl8Zn=eizCZcn- zt)aR8`gP%XZ_fYlZSFg_=kn=e71B3qL>8 zzvyCr{jjHQ(!rx&TUqikKgL7L{p}7u|NN@{)|Y+y*X`rSaA@V{t=GRT{Xq7A$lSdC z>3>W0_t{(S9(POsE#D08q@lI``uSbb|Hu8_-|DZo`unopUq7#-zmFUJ_48Kq-SYDT zKkwGQ#gD|$X8%)tw{Op1@BG}_=8yiZ-oM4Yp87wSKmYmf?}7jSXKcu?m!UmtzrTLM`g{57t^S7mMj7%O zWyo(7zn}WMaAX}@CszNeU9a8#_x|)w|Kt6ybbHU+;IFsbVD75^hx@y8{%Z&Q)4zTE zr2X1*|K{kU-7@_xddrXNkRQq+KZ-xE`_8|9hKBn0NdNxvV>9%Z)xQ=$9>uYVogWu&i%WOr zH{Vj%eo*fCAAH++j`G9vjvpQm|M0lt#oxP^-Qm=sZKU&Z z=xdG+J1-}f) zZ!vj3_u{$Qi|1c2o@>2${`BHGj`ce0O;1T)Jgs=~aQel=-WLyXUp%CJ@lf*R=`ZWK z^XehP>u+bBSARfX{h56AGw|vs^VPT7>z7l@WgOml{rc6Ku;#6O>%ejuhj3mU!ZG$~ zU09Eu*I}0*w%uWu_jv7fc)?n)5d6dd6-n+wtJ)wYz5RS+~|*=e6f+dp_Q?FFo7tx&EGQ z_x`YKXPj-^cE;`Jxa%0_AOGF5{c-+@0qcw9auc>QF=M$MCR}d9{!ZARiEozclNY>t zh5y$HUpMJCn0)BGPTBvdQLAe)PHkECZ^~^r<#w35vaT&(KW#szU3Qvj+I=z2J8fU4 z?dy!~&G_7m+uV=a>+G)Oy5?NR{B-Aa!Q~g`ERF@+U$oDQ?2B&mMcY|)U5l<`(dC!i zu1mJJWE;z_bA@Tec30e9Yi@%zx8a)WTz9{%dw<)yMxY-7*m_uN`P93{N%9ndg^ZQo8f|99v00r}z2&g-N7eB7~YX|LAUfw4Z(}y!s9B>gdy}qf4)U+`f(4lmkq!4ko=iko4*R(yIeFuMW<Oje>gCegEX1qFJ@%op|Iw0}tz{9J93$G3uym~_Z>WTQPC)ux_ zWWRbc{pyMI>v!Ap8{*ZI-d9g%Up&JZmeg`SsR_V?t*pQdGk~L)|;|?yk}o} ze_KzTw{iP4{=M@yVLM)r=550DPyE$+n;f$2&*VwxZOVR5eYNb%G}H8M=WT{>#^+`p zE!&x0wOrqv%g)>1dHdq0?rmYwI=6oBy!nZHTU@eMEZbWAX1T4FYE4wf@_#Y2B@3)b?%N<^7Dl zt-FQR-RkRf8*bSRo87Qy8*b4JSF>pkHeGJ()^h7_dwYA!I_ zb$9Oif?XT(1O2u?-+4P2?7SV?^&?+!?CoP$aO!hs{ylT$=WdpBj&nP4;qqRZxc?LU zcH!>4c(mLMm#*;A-FRsyF73pn9djDz?P|jEHCMLdr|#{Wx4+$W-u&%*l+4-Hx@t-T9h3$UB$6v;X(@{odW?<^A67UGKdc%dz*jN7v&t(A%Sr zKiU_^$KM>!esf&+%`w@xKi$y&mcBVQ`Q~`!n`4h}jw`-7M)&5J+?yYrH^x?A_z4_aSdPzw+)6>igF*>$3Ac zyxw_t0ONgRy7TT^{oS|v`{=QCYT4H4z4c)IWj%M^9f^1!bGb3w8gp4kBNP#xcSjrE z{V2RUy72C(!h6sAegxjfZO0#`_lZ9{?~`9FUq9)(Cwta`b<=sDvj6`0yifbMZ~gZf zmzl9&zWv{4t~>9u_IK9jW|uqfbBuF7H+R^1pZ7KMJ~qE)?OXPHo_*dn=Wi{y!My#R zcRMZEUoTt#zToqVu6uE|^X|vteVJu>!?KN)LCbZn@UFP+R^2|U%xkXGA0^F)b!7Rt zAB6Yy0n5Jo1NFYayWw*FNWE{m%$Cb<*}iZ1_Z_$Ij@xo~+?wgU@44KbxA*MZ{;B1* zapJT8{r~PS&HDlCf&DsIwCvk~>o~CQ2lnOAbskPwws|;Z*_XrLEnjfI=x$`kNX7#KK>!$PJ{N=|NpBq`W_N-&; z)bjZe+Zp-0^D%1s4(5G0i1#t-a(*s8Mz=a2j=Fs~>h|G5--jb@AHMfLy0+b&u{Nx0 z%Qm{-J0HCR>#*}N?&IUGXZ)e_F=1aP<}B||x;{PV$CUl^0PVvAw2!HE>&S8)9+-Vh z`}~a0&n$I5=3MujkImWsoPC^M?0hWjcRrRTEVqdVQyydps^7Y5IcWPUwZu3+7>3i|x z%;(Nr=FGmI+18n_J-3~6`*h*<_sHqv(rxtZvGZ|ln@+obTrXHId*k-Ku@5&)HypRV z?$$m!CI8`+{Ku`^`_8|9SAX2wuY2Dcem8$SxE&wdj(#_PJi5IfecZXI4-aQPobUN? zPUFLQj1T7|KAemA@NnhBarFqlJ9 z!;a5U+aGgXW47B{vX(oa<6kj(?$2Ad|Gn)!xLqIr?0h;m{plR)r-!$n&S8H3X>?{L9ySX#43r+o$trpZ-RFdU*T!w|#Yf?9=zkr*l)E&P9DX z@AT;$&Zl!YpU%g8IuG;dJWTVXr>%~4+WJwaZCrKQ=47XxKX%&1taWJl__yDz0c+42 zvc6bftzm1#8ntZWTi5DY_TOP->Z_}3R`DUBn=B#;Z!CJJItYvG(TD8`!b!)@g zw6-k!`EAGAwf3xi>%g*q-}?J^Y@JxA)|qu~U09bE)3{|)~Q{|)~Q{|&$2x$TDE@7;F8f5U&nf5U&n zf5U&nf5U&nf5U&nf5U&nf5U&n@3~03;lJVcIHP&I-8`#pewR1D%iAsgE&nb5Ex+IG z&F}Vh%WveW8M$g6pEr-sn`2__mj9OjmfvHR=5Ii|<-g_k_@&+Q-}2w`-}2w`-}2w` z-}2w`-}2w`-|~CB({B0w{%E)S9{V(pecCPmE&nb5Ex%)N?Uw(R|Caxj|Caxj|Caxj z|Caxj-{Yos%YVy%%YV!7cT>CNzvcIss@?IM>uGoVcl>w!cl>w!{-(7%{yY9V{yY9V z{yY9Vevil69seEw9seEw9lz%-&5TgH<9F1c-SPX|*vt$yGegbunRds2$A8D~Fw!j-fWsk=h;q9seEw9seEw9seEw z9seD_=TYsB|DNCTsphd{b7Z64^WXE|^WXE|^WXCuwQEN0no+xU&wtN<&+kY`yXW_E zl4dmP(5c4+tf_x%2TH2*t; z=DA+;T(3Fe((d{1`5k#__xv7bw0r)0{(Js={(Js=evdzz=Z5W`|DNBBR&%tb-Sgk` z-}684Kkz^BKkz^BdoI}?_#gNm_#gNm_#gNm_#gNm_#gNm_|1H^2mS~C2mS|s$Fkc4 z{{#O6{{#O6{{#O6{{#O6zsK$Ef&YR3f&YR3f&YR3f&YR3f&YR3f&YR3f&YR3f&YR3 zf!`6R=CM}u_qILoKkz^BKkz^BJ37@K_&pYD5B#1hvGjv*w6dd*pxQcjT-&a@HRCANe2oJ*RJv{Ez&P{Ez$|Pd1Myn}UNfuL%<46>dd;j}a}J@I)oYFzHnV!otX?y#*UaiQ=M|b+z2?|P zGppCk>NV#XnpwSOR&A6)oW(;nj?_StX^{-qM6leo*y=| zdd;j}GppA;S8QhWnpwSORPqR&8%KCtJloxHRn8&A6 z)oXw7o7HP(^_p3|W>&8`rqj&oHM4rntX^|8wVBmx4y87aGn-kxW>&A6)oW(;n!~Hj ztX^}7wVBmxX7!rKqRnxkW>&8`2h+^zHM4rnxtQi$Of#$3{^U2S*UaiQvwF?>nPyh6 znbm7%^_p3|W>&A6)oW(;nsYbJtX?y#*F4T`X7!p`z4j;npZtzNwLkg);Fk=Fj^H`~`7N^hQXG_!rpY+p0m*Z$@=+t(aoZ)W?N*}mpEN%NefneA&%!Zovf z&GVDyIASx~*E~;YfAgF1Yi9hK8NcS7W;5g0%=opx`TyqkT&9`zYmPlO#~zzmzh>62 zd0yPi`nA9L&H6RZk(*h+X4bEn^=oGRnpwYQ)~}iMYi9kLS-)o1ubK60j`cLNe$Dgh zX4bEn^=qCpHM4%rtY32+sG0R^X8oF3zh>62c|O(5`ZcqD?K}T>{_p%|{+j1m&CFjj z^ViJ$weS4j`JHNLjwdzG!J1PJ&Fo(@``5nnoBeCw`OW{e@BC%}o0s=)<^bDwezSnh z$;;+6Mf1GBnFVZS0h?LCW)`rS1#D&ko6{G~EMRjAqnQP4W&xXHwCy{;S-|$4-|?*G z^hPri*uL|d3vA!{%?38dxtil#?K{62!RG&s+P?FD=Xc_?nHOy41)F)nW?ry49@d;p zZRQ1=dBJ90usJ@~oH}Vvoixuio9CL%ykIjg*vtzy^MdV(-@IUR9IcrbY)-K>^McL1 zU~^opIj+$>KW*j(o8ueJ^VHUVo@!pOnHOy41)F)nW?rzF7i{JQn|ZLGcVX2BWdOZn|Z-zUa*-LY>t^U$4r`;!DeQ#nHg+m2Ah+@&D>yfe6Kz6 zJ3-u@`JegC5;muRnpwj3%F> z|IGi)Z{DyumDJ1|HuHwfykUFhf98MYf98MYH-p&BAT~3I?U~;kVl#)>%po>&h|L^g zGl$sBAvVY3n_0y6%>T^)%>T^)%x^xiJ@cDSY))S_Gm6cOVl$)IoXTpBXSQd4GX>56 zBeyxO-=3d4`RzArz#6oMtS{DAYuFmGMy)ZcYxS&gYr>kermSge#+tR}ta)p}TC|p| zWoyM+wbrb4Ys1>KwybSy$J({_tbOaiI<$_gW9!5^wa%<_>%zLUuB>m?wRL0NT6fmH z^iESQ^l#|n3jsUo|u-3$*7o&iXoMlhl;tUn1_lX zrxfTAj((%ul!oEXvLxx%UAxd{9pOM z@_*(3%5PjC8nS4}q9KdnqZmGlhAak|E=YliLuVu?H|1iI% zEyMi7{6-}*%sbd_O#gI5{m zALbwCALbwCALbwCALbw7AK@S2AK@S2AK`a?BpSMC=wf&)Bm5)$&Y8qOew@#ab0=|5 zJI4GRp5QGT-q8RJ(#WQ>1|UkQ;h zeuH;0hmbLTRYb=4jpSsE-#}i*_{aFoDP)X)jNiCU#`wqh$N0zi$N0@LWQ^Y|L&o@x z^kj^GjDL)OjNgny#`wqh$N0zi$N0zi$N0zi$N0zil^Jo4IL;Agj9-}%=ZZ7Nuh57> zBhKT+z+jv|j`PPE;~(R9ZZBi}WBgg zUH&eAm%q#3gUH&eAm%q#3<#*mZUH&eA zkH5#?1}%^!Ug5%?@Xrf1Ka>-WV8-fzcQ%%Q*iy|2Y3R z|2Y3R|2Y3R|2Y3R|2Y3R|2V%v(~R>QHqAKyIKQFOjPsB4n^nj-|2Y3R|2Y3RzxjoX z^N;h7^N;h7^N;hJ@yR&9mBMl;5Sg43H}NG3H}NG3H}Lw^BS4ppWvV1pWrvj7X!GN;Gf{1 z;Gf|4znjhkzroy0@K5j?(9HzDS*T3#Pw-FhPw<yD8IjT(Zo2AMmzj>-m@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l z@=x+l@=x*`Ps}9$B>yD;B)^%hO!80iPx2dI%q0IL|0KVemQ3B>yD; zB)`GrO!80iPx4RlPx4Rln}^9H|0Mqu{}lfezd`00lZ-LROz}_g8*Gl*nV6l)6u{}lfezhUZ3@lWwj@tfbt6u%jsOz|7A&J@4be$N#D6#o>z*^Nx`Pw`Lj zPw`LjPw`LlPxDXnPxDXnPxG6V%QU}X?U=sVN=)BMx?)BI-gGR;5DKg~bQKg~bQKg~bQKg~bQKg~bQ zKg~bQKg~bQKg~bQKg~bQKh1BrJci3N%|FdQ&2Qc*)BMx?GyF6BGyF6BGyF6BGyG-* zGsAB-Dl_~u{4@MB{4@MB{4@MB{6=Xr!#~46!#~46!#~46!#~46!*Av=GyF6BGyF6B zGyF6BGyF6BGyF6BGyI0(V@5GE{92C8@Xzqi@Xzqi@S9)E4F3%O4F3%O48O77%<#|f z&+yOi&+yOi8~4o&|1AG3|1AG3|17^*$jtK3^3U>{i_9$lEWZKX%<|9j&+^am&+^am z&+^am&+^am&+?n6%q+jr;>_~T^3U=c@Xsv&EdMOOVgJnX&+^am&+^am&+^am&+^am z&+^am&+^am&+^am&+^am&+^am&+^am&+_YbGRv>s$t=I9AhZ0l{B!(s{B!(s{6?QM z$3MqE$3MqE$8RPybNqAsbNqAsbNqAsbNqAsbNqAsbNqAsbNqAsbNqAsbNuE?Gsi#2 zKgU1EKgU1EKgU1EKgVzOG;{oO{B!(s{B!(s{B!)~^fJdk$8TOQbNpuZGRHs1FPg|4 z{~Z4u{~Z4uznQ+w@z3$k@z3+m^DA&O&p*#U&u{KE^ZfJt^ZfJt^ZfJt^ZfJt^ZfJt z^ZfJt^ZfJt+NhWt%sju@+066L^Uw3o^Uw3o^P8#7JpVlZJpVku8QaYB&-2goo43t8 z|2)5rDf9gE{PXmQ^ZfJt=5;gAzrer1 zzrer1zrer1zrer1zrer1zrer1zrer1zrer1zrer1zrer1zrZh0iIMp%@GtP23(f-n z0{;TP7Ap(<3;YZGx~welFYqt$n<35u{{sI4{{sI4{{p`;{4DS<@GtN$@GtN$@GtN$ z@GtN$@SD}k0{;U40{;U40{;U40{;TPS-vdtFY+()i)ymSzsPUYKa2c}{EPf%{<6rw z$iK+H$iK+H$Zys;i~NiHi~NiHGJ!1eO9isXzsSGHFV@K-|04e)|04e)|02JnAdCEq z{EPgH{EPgH{EPgH{N@g`$iK)hKgc5gBER@2i~NiHBA}Q}%p$)$A&dNr{EPgH{EPgH z{7d{x{9>Uj@h|a98M4H`#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9 z#J|M9#J|M9#J|M9#J|M9#BWYKvWYD5FYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz& zFYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz)FY_<+FY_<+FY}x8%`*Qo|1$qF|1$qF z|1$qF|1!VX_AK+8369Pq%lymy%lymy%lymy%lymyW<_HbKFj>(;iC)5GXFCFGXFCF zGXFCFGXFCFGXFCFGXFCFGQWA#Eb}k(FY_<+FY_<+>l(Anzs$ePzs$ePzs$ePzs$eP zZ+kG2NZw@yr{44w`{AO~aJID(E3jZqqD*r0~D*r0~D!;kitn!=V zjv3Od@~`r*@~`r*@~`r*@{4$~%D>9L%D>9L%D>9L%D>9L%D>7l_Q@*$D*r0~D*r0~ zD*r0~D*r0~D*r0~D!)1Mtn#n&ukx?*>n5_wzskSLzskSLFC@w;zj@=BH;#GZm^aQU z|0@3~|0=)kHM)ze@vrf(@vrf(@vrfVk+R0W#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD z#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD#=pjI7CL63v&O&1zs9cviP`9^@vrf(@vrf( z@#{mf#xI!48vh#q8oy>FYy9S^v&Ju^${PO~{~G@q{~G@q{~Eu@DjJil^RM%-^RM%- z^RM#@up+?9I{!NVI{!Mqnf|Qvuk)|-uk)|-uk)|-3o9eq$~yl#|2n^bE9?C0{F;_% zTC&bB>dHF5+3&3Luk)|-n_G@hs_;>hs_;>hs_;>hs_;>hs_%%w|;n&+` zhku8EhhI~g9sV8u9sV8u9sV8u9e(k6cKCPrcldQF+2PkaWru%A;c94Dkeg1v^ zeSR-~kbQnHfRKIueg1v^eg1uZFN%Myfd7F1fd7F1fd7F1 zfd7F1fd7F1fd7DBLmRK!kOTe$ey`q;1Abj?4)_oF5BLxG5BLxG^|m?SKj1&$Kj8O@ z4>{o1G)9M;1O5a41O5a41O5a41O5a41O5a41O5a41AeajLw;>t4*3uH5BU%I5BU%I5BU%I5BatCIpja&Kjhcr<&gi7|B(NXU!Rvl zeob-?`49OI`49OI`49OI`49OI`49OI`49OI`Mq#+4*9jrIpja$KjPQ?<%s`?|A_yH z|A_yH|A=4roFo1t{v-Y){v-Y){v&?fX^!}h_>cIH_>cIH_>cIH_>cIH_>cIH_>cIH z_>cIH_;sr};y>a);y>a);y>a);y>a);y>a);y>a);@2wXi2sQHi2sQHi2sQHi2sQH zi2sQHi2sOR+n6JMootTzz1~5N`H%UJ`H%UJ`H%UJ`H%UJ`H%UJ`H%UJ`H%UJ`H%U% zUP6xfkNJ=JkNJ=JkNJ=JkNGv@Ip#m+KjuH?KjuH?KjuH?KjuH?_lhSu=0D~?=0D~? z=0E25no2q5KjuH?KjuH?KjuH?KjuH?KjuH?KjuH?_uAY!=0D~?=GU+1g#U#9g#U#9 zg#U#9g#U#9g#U#9g#U#9g#U#9g#U#9g#U#9g#U#9g#U#9g#U#9g#U#9g#U#9g#U#9 zg#U#9g#U#9g#U#9gx{;Y>%zu>>%zu>>%_gXT!;J@I%;MZqIpPdW-3;qlK3;qlK3;qj!&3P{P zFZeI`FZeI`FZeI`y(UdA_`NPoF8D9_FZeI`FZeI`z2-qK`7ik|`7imsdQ>j?FZnO| zFZsPLLN56)`7ik|`MsV;F8MF{FZnO|{jUph$$!c36{vE_f60Hzf60Hz?{)8T$?vst za>;+mf60Hzf60Hzf60Hzf60Hzf60Hzf60Hz?=^OE$$!az$$!c3HFt8!f60Hzf60Hz zf64E)c;dBqa>;+mf60Hz@AXJ>#ec=`)v|KMf5q=LN^-@2#ecakef5m^r zf5m^rf5m^r?=^mM#ecake@3nt&#eci8W#ecakef5m^rf5m^rf5or= zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFhy|Aj4n{eSfT z(f>#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz

#PAN_yy|Iz#PAN_yy|Iz#PAN_yy z|Iz#PAN_yy|Iz#PAN_yy|Iz#P zAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz

#PAN_yy|Iz#PAN_yy|Iz#PAN_yy z|Iz#PAN_yy|Iz#PAN_yy|Iz#P zAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz

#PAN_yy|Iz#PAN_yy|Iz#PAN_yy z|Iz#PAN_yy|Iz#PAN_yy|Iz#P zAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz

#PAN_yy|Iz#PAN_yy|Iz#PAN_yy z|Iz#PAN_yy|Iz#PAN_yy|Iz#P zAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz#PAN_yy|Iz

#PAN_yy|Iz#PAN_yy|Iz#PAN_yy z|Iz#PAN_yy|Iz#PAN_yy|Iz#P zAN_yy|Iz|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|24?3 z|1bT2>HkarU;6(VHkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Z*ATz{zx4m5|1bT2>HlkpU;khF|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarUtRt#f0w_@-{tS}clo>gUH&eAm%q#3HkarUp@XFe~-V%um3OofA#qF|JCE~@%Q+9{5}32e~-V%-{bG`_xOAK zJ^mhlkH5#?@A3Eed;C5A9)FL&$KT`c@%Q+9{5}32e~-V% z-{bG`_xOAKJ^mhlkH5!1&OgpS&OgpS&OgpS&OgpS&aeM3{eS8IYn*?af1H1uU;khF z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hlkje}aF4e}aF4e}aF4e}aF4e}Z5C zU;6)=;Gf{1;Gf{v|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|Nqy<{q#tl7-k)}IW8BpZ-5G1trZ6Z2qEPV3B?jYNLg`> zn%%Z{HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ>p3?f=w%{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK@&h5aw=*Z-&gPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?@7Mi5OBjGK0AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K; z2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L& zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy zy0QPp{u}#m?8g9v0qDm58~bnUzp?+u{u}#m?7y-9#{L`oZ|uLZ|Hl3s`)};OvH!;Y z8~ZT;VF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp22t z+yC1B*Y>}*9|I5uAPhhlfL`1G+Wyz}zqbFi{jcqRZU1ZgU)%rM{@3=uw*R&LukC+r z|7-hi?Z37E*8W@jZ|%Re|JMFn`)}>Pwg1-sTl;VAzqS9?{#*NR?Z37E*8W@jZ|%Re z|JMFn`)}>Pwg1-sTl;VAzqS9?{#*NR?Z37E*8W@jZ|%Re|JMFn`)}>PwI2fz2B2H} zZ|%Re|JHsCKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXpga5T?7y@B&i*_5 z@9e*`|IYq9`|s?>0E7YP&i*_5@9e*`9|I5uAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaPQz{yY2c?7y@B&i*_5@9e*`|IYq9`|s?(v;WTiJNxhKzq9|&{yY2c?7y@B z&i*_5@9e*`|IYq9`|s?(v;WTiJNxhKzq20$5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE)&*I%fuqX9$%hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{y+?APxhbeKiPk>|78Ek{*(PD`%m_t>_6Fmvj1fN$^Mi5C;LzK(*Q;T z7!6=FfYAU(0~ifpG=R|nMg!Q%{*(PQfYAU(0~ifpC;LzKpX@)`f3p8%|H=N7{U`fR z_MhxO*?+SCWdF(jll?S+(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PgJyej31N0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1( zqXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}H zXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXF#S+fM@+4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y^S||vt;1(uJO8Pt3?2T{Rg)xqO14o3%%SUCLPa`= z@7>{#E{8vM@I$}D|GFIh+`-QQ4wsJ!M9)9=)RV)fmxBxcv1gwgUS1BbE{D%Aho3ok z8K%P*m&481&OiCVfAGu6e)5C=_*dTWj%)elNB!>F{^dLW?N{IV@4x!a&%N`9|HDZy zXZhw0?_AF5m+xH8=@0+sS0}lD6@&$H)`{lj?vm;rG9SfG6VSY~(rSdVi4#Q#LjpZM$d{E5GN&Y$QZ&!32T z&YyT0@c9$3Q*>}6ed<{who3rl9Pa$7zW~plYU&+a?x(V$^QSt6^9!#aIKS|4?D>Vq zz0NN@SmfZkyzsD#!;c*N$M*cfzXInMLh$nox!L)Je(C(;=e~b0ykz+N!fUP0FT8l> z{K6|x&YyY6{QQ~6LeHOhaOnJ*pH|MF`Qv~7j7o6+Obc-S%q!E+pLvOigA0G@3I6j- z&z3s;nShwFLlfthe)c%O^bgkgrGE*|FO{?Bmy)#e%m3msUV6p+`K6bA zpI>^7%=wikcFwOnOL2bX>4NhskM*8kd2sRk%K!Mzul$$h{L0U;=T~S82UqRYt@C?z zxoWSxQ2hMLt9s9`ysYW`%IjFppL-(t{JBS)&Y%0||NOb?{rtHm`24w7!k<6)60`H? zUiWnVnIAYE++aWRSK;|*GLQ4myx{lzg=a6%UwCTi{DtRS&R=-o=KO{Kji0~pBk}o* zTlc~nPhOvIJj;8&@ifl)#^d+r8xP=}Z~Qd)xBl;6f9u`14j8mw{NP9a`@&O4{2cuD z7k~B#ppaku@Q;7-!_VG%`|mi#!ATBIbNTGya;pFH<9~Vowv!&5_Ta>ReRg=qC*Jzs zUw`W#|K{WG9zOi&y$=rGef;=+hmRhiaYQ^~)E$^=E(Wtta1net7oLlP8a#K7V$2{=xO(!^h8_AKrWX;fIeO9scP*`Dcgc zSI@rl>>EyF>6Mc_{?7HI2j9PX{^0Q=AH9C`{QA***WUQ{hu;43!&i^qKRmkn@cR9O zk3D$)y~FkIKYx1l-sNlFdHc`5{>|r?tN-lq(c#IHg9|=9yS(qhFTY(b|MI)5aQN`> z_|fHU*W>;amn-lcU-*stxz6Ct&E=zSKKHzg*f^r~K+_UQYI_SM|%! zyS|stxXOp?`?s9_;Cf#^c>Vi_%XNQqcK6@>pM3qTfB5e8Gq=}=AGxibJbnCo*M}!p zPcK8i`{4Np_c!Cy>u+5@y}xN5eE8(y^)GLnci#S;cMrb!&F0G)TrR|U9DeV9zVrF& zk}v0V`Q(>3%+qV%=Ievs+~AjwKR8^@`EWVwujbxgzrX+1-}sHIhYx+?@Xco*e0X*J z@SV5+_HP`n9v&WEzRZU&zkSo~_<>va%6UF~bFcn`ufO$;cijb^UOjLlK05sNlgk^= zo?m^>RBt}~-sPj0JIRxW-hJoocb)ojUQe%l;{Gb#AAj4e@#P2HVO+xZzFd1byNBNU z%aeY&?aATc|MZW(@ax8TJ$(H9*WUg|-@HGM zhhN=l_b0rZ>zCij%jqv)baQ{Q`#%>>`~2{0m-pZLldr$^ci;V$`|;s-Ke~E&`R=`c z{p9-5`_J6#?pON0KX~-u(YM_fpFa8M**kCl^LPDOeRCIk^C#ePk-zJH?6&>#?()7H z$9L%P{+r2f?!RBIygxrPzkT_#H|P0gv3u0t`>n_CUq3v2>%qh8XNN1_KE1wr-&K0=w_GI`djGAxeD#~lb=u2?7=7icytz|9 z{_2kN<&AXt{=2um`Q+8%y~9&y_3X{pzd3*JeRYrj2fo#Z-+TPt)xY@Y;nmacxj#I4 zcDTOsm%@_=*RJAomvp()Jiq?-wR`aUmp}O5dT{;powu*Pyp#OO7hkTLGk9}RmovG1 o;LZ75-g^G{a=daYUVgva<=jWVeYk#l|L$e2Gj~(@$o=*Ee|IW)M*si- literal 0 HcmV?d00001 diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.fixtures.tsv b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.fixtures.tsv new file mode 100644 index 0000000000..5eeffe47e5 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.fixtures.tsv @@ -0,0 +1,40 @@ + 0 + 0 + 0 +a 1 ▁a 10 0 1 ▁a +Hello world 7 ▁ 5 0 0 H 241 0 1 e 8 1 2 ll 105 2 4 o 17 4 5 ▁wor 160 5 9 ld 74 9 11 ▁Hello▁world + Hello world 7 ▁ 5 1 1 H 241 1 2 e 8 2 3 ll 105 3 5 o 17 5 6 ▁wor 160 6 12 ld 74 12 14 ▁Hello▁world +Hello world.\nSecond line\ttabbed 23 ▁ 5 0 0 H 241 0 1 e 8 1 2 ll 105 2 4 o 17 4 5 ▁wor 160 5 9 ld 74 9 11 . 7 11 12 ▁ 5 12 13 S 297 13 14 e 8 14 15 c 38 15 16 o 17 16 17 nd 24 17 19 ▁ 5 19 20 l 30 20 21 ine 147 21 24 ▁ 5 24 25 t 11 25 26 a 13 26 27 b 45 27 28 b 45 28 29 ed 18 29 31 ▁Hello▁world.▁Second▁line▁tabbed +The quick brown fox jumps over the lazy dog. 28 ▁Th 25 0 2 e 8 2 3 ▁qu 69 3 6 i 15 6 7 ck 43 7 9 ▁b 47 9 11 r 23 11 12 ow 60 12 14 n 9 14 15 ▁fo 97 15 18 x 108 18 19 ▁ 5 19 20 j 115 20 21 u 14 21 22 m 26 22 23 p 27 23 24 s 6 24 25 ▁ 5 25 26 o 17 26 27 ve 102 27 29 r 23 29 30 ▁the 12 30 34 ▁la 88 34 37 z 54 37 38 y 19 38 39 ▁do 159 39 42 g 48 42 43 . 7 43 44 ▁The▁quick▁brown▁fox▁jumps▁over▁the▁lazy▁dog. +tokenization and segmentation 4 ▁tokenization 200 0 12 ▁a 10 12 14 nd 24 14 16 ▁segmentation 80 16 29 ▁tokenization▁and▁segmentation +Antidisestablishmentarianism 14 ▁An 123 0 2 t 11 2 3 i 15 3 4 d 33 4 5 is 22 5 7 est 63 7 10 a 13 10 11 b 45 11 12 lish 193 12 16 ment 209 16 20 aria 221 20 24 n 9 24 25 is 22 25 27 m 26 27 28 ▁Antidisestablishmentarianism +water running walked faster apple book work play 15 ▁water 70 0 5 ▁runn 211 5 10 ing 20 10 13 ▁walk 87 13 18 ed 18 18 20 ▁fast 85 20 25 er 16 25 27 ▁appl 214 27 32 e 8 32 33 ▁b 47 33 35 o 17 35 36 o 17 36 37 k 66 37 38 ▁work 57 38 43 ▁play 71 43 48 ▁water▁running▁walked▁faster▁apple▁book▁work▁play +3.14159 x 42 = 1024? 21 ▁ 5 0 0 3 225 0 1 . 7 1 2 1 109 2 3 4 110 3 4 1 109 4 5 5 238 5 6 9 239 6 7 ▁ 5 7 8 x 108 8 9 ▁ 5 9 10 4 110 10 11 2 169 11 12 ▁ 5 12 13 = 0 13 14 ▁ 5 14 15 1 109 15 16 0 282 16 17 2 169 17 18 4 110 18 19 ? 170 19 20 ▁3.14159▁x▁42▁=▁1024? +!!!???... 10 ▁ 5 0 0 ! 113 0 1 ! 113 1 2 ! 113 2 3 ? 170 3 4 ? 170 4 5 ? 170 5 6 . 7 6 7 . 7 7 8 . 7 8 9 ▁!!!???... +(parentheses) and [brackets] and {braces} 27 ▁ 5 0 0 ( 236 0 1 p 27 1 2 are 34 2 5 n 9 5 6 th 36 6 8 e 8 8 9 ses 150 9 12 ) 237 12 13 ▁a 10 13 15 nd 24 15 17 ▁ 5 17 18 [ 0 18 19 b 45 19 20 r 23 20 21 ack 89 21 24 e 8 24 25 ts 101 25 27 ] 0 27 28 ▁a 10 28 30 nd 24 30 32 ▁ 5 32 33 { 0 33 34 b 45 34 35 ra 152 35 37 ces 216 37 40 } 0 40 41 ▁(parentheses)▁and▁[brackets]▁and▁{braces} +café naïve fiancé résumé 21 ▁ 5 0 0 ca 104 0 2 f 41 2 3 é 247 3 4 ▁ 5 4 5 n 9 5 6 a 13 6 7 ï 0 7 8 ve 102 8 10 ▁fi 210 10 13 a 13 13 14 n 9 14 15 c 38 15 16 é 247 16 17 ▁ 5 17 18 r 23 18 19 é 247 19 20 s 6 20 21 u 14 21 22 m 26 22 23 é 247 23 24 ▁café▁naïve▁fiancé▁résumé +financial fluid 13 ▁fi 210 0 1 n 9 1 2 a 13 2 3 n 9 3 4 c 38 4 5 i 15 5 6 al 21 6 8 ▁ 5 8 9 f 41 9 9 l 30 9 10 u 14 10 11 i 15 11 12 d 33 12 13 ▁financial▁fluid +① ⑪ ㋿ ＫＡＴＡＫＡＮＡ 16 ▁ 5 0 0 1 109 0 1 ▁ 5 1 2 1 109 2 2 1 109 2 3 ▁ 5 3 4 令和 0 4 5 ▁ 5 5 6 K 0 6 7 A 296 7 8 T 299 8 9 A 296 9 10 K 0 10 11 A 296 11 12 N 224 12 13 A 296 13 14 ▁1▁11▁令和▁KATAKANA +ｶﾀｶﾅ half width 11 ▁ 5 0 0 カ 0 0 1 タ 227 1 2 カナ 0 2 4 ▁ 5 4 5 h 32 5 6 al 21 6 8 f 41 8 9 ▁wi 73 9 12 d 33 12 13 th 36 13 15 ▁カタカナ▁half▁width +東京タワーへ行きました 10 ▁ 5 0 0 東 231 0 1 京 230 1 2 タ 227 2 3 ワ 228 3 4 ー 229 4 5 へ行き 0 5 8 ま 287 8 9 し 179 9 10 た 256 10 11 ▁東京タワーへ行きました +日本語とEnglish混在 9 ▁ 5 0 0 日 264 0 1 本 266 1 2 語 269 2 3 と 0 3 4 E 295 4 5 ng 86 5 7 lish 193 7 11 混在 0 11 13 ▁日本語とEnglish混在 +Привет мир 11 ▁ 5 0 0 П 249 0 1 р 112 1 2 и 111 2 3 в 172 3 4 е 173 4 5 т 252 5 6 ▁ 5 6 7 м 174 7 8 и 111 8 9 р 112 9 10 ▁Привет▁мир +안녕하세요 세계 9 ▁ 5 0 0 안 234 0 1 녕 233 1 2 하 235 2 3 세 181 3 4 요 274 4 5 ▁ 5 5 6 세 181 6 7 계 271 7 8 ▁안녕하세요▁세계 +你好，世界！ 7 ▁ 5 0 0 你 261 0 1 好 262 1 2 , 31 2 3 世 259 3 4 界 267 4 5 ! 113 5 6 ▁你好,世界! +I love 🍕 pizza 12 ▁ 5 0 0 I 294 0 1 ▁lo 53 1 4 ve 102 4 6 ▁ 5 6 7 🍕 279 7 9 ▁ 5 9 10 p 27 10 11 i 15 11 12 z 54 12 13 z 54 13 14 a 13 14 15 ▁I▁love▁🍕▁pizza +flags 🇩🇪 🇺🇸 end 14 ▁ 5 0 0 f 41 0 1 l 30 1 2 a 13 2 3 g 48 3 4 s 6 4 5 ▁ 5 5 6 🇩 277 6 8 🇪 278 8 10 ▁ 5 10 11 🇺🇸 0 11 15 ▁ 5 15 16 e 8 16 17 nd 24 17 19 ▁flags▁🇩🇪▁🇺🇸▁end +family 👩‍👩‍👧‍👦 emoji 10 ▁famil 185 0 5 y 19 5 6 ▁ 5 6 7 👩‍👩‍👧‍👦 0 7 18 ▁ 5 18 19 e 8 19 20 m 26 20 21 o 17 21 22 j 115 22 23 i 15 23 24 ▁family▁👩‍👩‍👧‍👦▁emoji +zero​width and non breaking 19 ▁ 5 0 0 z 54 0 1 er 16 1 3 o 17 3 4 ▁wi 73 4 7 d 33 7 8 th 36 8 10 ▁a 10 10 12 nd 24 12 14 ▁ 5 14 15 n 9 15 16 o 17 16 17 n 9 17 18 ▁b 47 18 20 r 23 20 21 e 8 21 22 a 13 22 23 k 66 23 24 ing 20 24 27 ▁zero▁width▁and▁non▁breaking +quotes “fancy” and ‘single’ — dash 24 ▁quote 198 0 5 s 6 5 6 ▁ 5 6 7 “ 0 7 8 f 41 8 9 a 13 9 10 n 9 10 11 c 38 11 12 y 19 12 13 ” 0 13 14 ▁a 10 14 16 nd 24 16 18 ▁ 5 18 19 ‘ 0 19 20 s 6 20 21 ing 20 21 24 le 107 24 26 ’ 0 26 27 ▁ 5 27 28 — 0 28 29 ▁d 100 29 31 a 13 31 32 s 6 32 33 h 32 33 34 ▁quotes▁“fancy”▁and▁‘single’▁—▁dash + the [URL] token 9 ▁ 5 0 0 3 0 6 ▁the 12 6 10 ▁ 5 10 11 [URL] 4 11 16 ▁to 51 16 19 k 66 19 20 e 8 20 21 n 9 21 22 ▁▁the▁[URL]▁token +a b[URL]c 6 ▁a 10 0 1 ▁ 5 1 2 3 2 8 b 45 8 9 [URL] 4 9 14 c 38 14 15 ▁a▁b[URL]c +control tokens inline 20 ▁co 99 0 2 n 9 2 3 tro 128 3 6 l 30 6 7 ▁ 5 7 8 < 0 8 9 s 6 9 10 > 0 10 11 ▁to 51 11 14 k 66 14 15 e 8 15 16 n 9 16 17 s 6 17 18 ▁ 5 18 19 0 22 23 ▁in 35 23 26 l 30 26 27 ine 147 27 30 ▁control▁▁tokens▁▁inline +https://example.com/path?q=1&x=2 29 ▁ 5 0 0 h 32 0 1 t 11 1 2 t 11 2 3 p 27 3 4 s 6 4 5 :// 0 5 8 e 8 8 9 x 108 9 10 a 13 10 11 m 26 11 12 p 27 12 13 le 107 13 15 . 7 15 16 c 38 16 17 o 17 17 18 m 26 18 19 / 0 19 20 p 27 20 21 a 13 21 22 th 36 22 24 ? 170 24 25 q 298 25 26 = 0 26 27 1 109 27 28 & 0 28 29 x 108 29 30 = 0 30 31 2 169 31 32 ▁https://example.com/path?q=1&x=2 +UPPER lower MiXeD case 19 ▁ 5 0 0 U 293 0 1 P 156 1 2 P 156 2 3 E 295 3 4 R 242 4 5 ▁lo 53 5 8 w 78 8 9 er 16 9 11 ▁ 5 11 12 M 288 12 13 i 15 13 14 X 0 14 15 e 8 15 16 D 289 16 17 ▁ 5 17 18 ca 104 18 20 s 6 20 21 e 8 21 22 ▁UPPER▁lower▁MiXeD▁case +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 10 0 1 a 13 1 2 a 13 2 3 a 13 3 4 a 13 4 5 a 13 5 6 a 13 6 7 a 13 7 8 a 13 8 9 a 13 9 10 a 13 10 11 a 13 11 12 a 13 12 13 a 13 13 14 a 13 14 15 a 13 15 16 a 13 16 17 a 13 17 18 a 13 18 19 a 13 19 20 a 13 20 21 a 13 21 22 a 13 22 23 a 13 23 24 a 13 24 25 a 13 25 26 a 13 26 27 a 13 27 28 a 13 28 29 a 13 29 30 a 13 30 31 a 13 31 32 a 13 32 33 a 13 33 34 a 13 34 35 a 13 35 36 a 13 36 37 a 13 37 38 a 13 38 39 a 13 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Ω≈ç√∫˜µ≤ 4 ▁ 5 0 0 Ω≈ç√∫ 0 0 5 ▁ 5 5 5 ̃μ≤ 0 5 8 ▁Ω≈ç√∫▁̃μ≤ +مرحبا بالعالم 4 ▁ 5 0 0 مرحبا 0 0 5 ▁ 5 5 6 بالعالم 0 6 13 ▁مرحبا▁بالعالم + leading and trailing 13 ▁ 5 2 2 le 107 2 4 a 13 4 5 d 33 5 6 ing 20 6 9 ▁a 10 9 11 nd 24 11 13 ▁ 5 13 14 t 11 14 15 ra 152 15 17 i 15 17 18 l 30 18 19 ing 20 19 22 ▁leading▁and▁trailing +\ttab\tstart 5 ▁ 5 1 1 t 11 1 2 a 13 2 3 b 45 3 4 ▁start 191 4 10 ▁tab▁start +newline\n\n\nruns 11 ▁ 5 0 0 n 9 0 1 e 8 1 2 w 78 2 3 l 30 3 4 ine 147 4 7 ▁ 5 7 10 r 23 10 11 u 14 11 12 n 9 12 13 s 6 13 14 ▁newline▁runs +mid spaces collapse 13 ▁ 5 0 0 m 26 0 1 i 15 1 2 d 33 2 3 ▁ 5 3 6 space 127 6 11 s 6 11 12 ▁co 99 12 17 ll 105 17 19 a 13 19 20 p 27 20 21 s 6 21 22 e 8 22 23 ▁mid▁spaces▁collapse diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram.model new file mode 100644 index 0000000000000000000000000000000000000000..b6e30611e409fcb4ae76b41b8abe0630ecac2f14 GIT binary patch literal 245202 zcmZU*3s{ubxv>4r3vbWRks{sjWupVjWujzjoVNpN)Zu(7;BIj z&O|LCNQ^Pon8|P&h5?7+G%youj9Q1PHPrYSwz7#;xB3xl+{PNW|Gj1ud;K5Rbzjf3 z*1OJYJ?r$o!=TVXfsvW(lA|@@?{WR0A%hH&>GxfRK_Pl_5ISHG zv6sjF>ZKL_*a3qE4Kn=K`zFoa6!6)g0fThu+QC<*`?Q~EN4}5sX^C1)-OE00oi?F! z<7a~g4+Ksg*wQj!#R{>_fMH zHfZ1=ec}eq;Y20dRBipx?S9Wy+QFT__jzv827R&PGgZQ*b*r^6ZtU^}h+nHMc=>*J@)v^=r>)JEJT}dXyrM{v4<3~1M==cn6aHicCoOJ{uDN7_5X1BN|>x=yxsmF zNa8x}f8OnPQNGoG?{}qTYF|Cp@t`gn<1-kJpO3=BU;OoJ<2PwvJcV18He-`^G#1@o z#EnVXnWeZW``T67AC3M%@$1)XTV8fQX#NDP@Vn3YW5jD2yAJvaxKZ2t^UojnGfo}} zI`r9~#|DMbpy`RLQxex@#QU1^%aQmG9W+q+Z%X-1>RQd`^Pq84SNYv^%1c_83hVEb z^+|~diP~S4hd<~W@ig0Q<)*mUQRox>k2c1yP1gQZx&8H88?POoi;Eh^CnRgSHT_Q* z=nI;0`As}?U6S_QA90dm<_4|zFa6=x$7>gU_k}No)tJLemFWJolQOg|P5vkGSlvKZ z|C2XTwZzYH`Dc}vwdOYuKZul&su})%E+N_Vg4{HjS)jVCO{*t`0L3?KDasPuvtsn!HMtn>Adxc+{s{QVBzxJH= zw$iBOiI#r#q|vVuXRMa3p40Ga;y1)6s1fAz;L&gWnm@<5b$*w>Hxp7ftj|pUXP=b# z4apDUJ~}9b9$%NbAtioo((fMl(j&db9Df{ti1iy%e-rT(s}%4e=?OQ#tMr^7y0;+JQs3?^|@@hSiCh?F)b0^!4$i^xhF% z`7%s}Qj;g#Hj_yzLjScZy&dWGS zQcX3QqeC2D`0_Lm2xOV89ol5r6|bwh?0kmc9YnceiC-uEZQ?;kzo%Z)PT*-;6; zp1EmvY}bQ?QWCX&yT9^1iO)>XMxQ+68hi?8KU(|R!4}N{WHB;7WcmJdNLEKfT zTGIhs{)V9P+CSd-PhX^@1TE?dRPsy0P{MEdz2A64yZpa?Z#90Nb)NI3oA@TfZO~^p z7#&H!^G`$9m79h-_$g+A>K{|{QZNEsjP#!}D*alFUsHh_`qdNrAF9rWYWqXg?N|NA zk+Eh&V*Dz<&+tJ*=!cBdwkN{uwTEL z*W%axC10`zdm$bFfX_%up{slz_bawm%X|3vjrf$LwOZjrkMzX&^flUU<>4deZ83+ULsgU|wfhJ9*UqFeRSZHuxBh`}NwOJU;dId!%KiGKyz)_DA0kzb;vO z^*C<-+H=2|{hj`!6a5j@pxC7S>?9sE@){QWqe}O;*2Xp3U%GJJZ;SQo*V3AQJL?ad z62DIS?bm)iiCxEQm;8GCx_B-7TXcVg)gI@Q?{HFKwfbrA8O}dQ|BWOqcOWi*jW=Yj zTc=e$=6{-)mWc(`D3|}~+EgZ-+>!lmmgBYBpGNybGmh3IX@CAPE^3iLDI=fh51FAh z7GM7qH$_;lY&UwQKVV`)hIV5%y1%+9saj^FKUn;RqDa9|H_YG3I8 z?$ds$Jr#Jzr%hvpyXVWg*JORD=Xf6Kx%Pk3+xzuz+G_jtzwK%0*DvmJ_3Kyi%e4*n z)4y6?)URJF|G=+Ps;995N`N-8Uz^^q&Fa@?_iHcqYrpE(mijei;mW{BP^lk3RI47U z)eluyzj|WtLsiWY{uEA{9;#}Q^EcIh?fJA{?=?O2?=|iEC;$5i`X+$7hyFPa!++EA ze!u@WEw+bx$Nhp6_kyLP-~U2=X}|urotB6C2lj{hN45R>#qw5 zVVVEik=O5kWoywt=@09B^}`+g{#XBy->>(zl=yW<>bH9f@3*nvQS(nvwwM2t;{)SA zIX=AaU?e=N72F>ai?wYJI&*KodZN*^)q)2DG$R7rZaUXmNx#2_}Nj#V_MvPNnKE!}=D*jMOxMF0~s|JaD z(I5*3i7|SOL3H^PL8lYLb`gECNMN~0K&eRZ9+9A3A_J>M2J90VY!ew|5eaD(8PXvV z2BGM~4v7psATqpDCr;QIGXQhD&lQ$n7Dw|F`CU1Q=RF;y* z@4kCXQjx16J4+`Y3=!iTqh7upEXKKyi~RgEk$EqQyzsin{C9)o7kN5a@N|&8xK<|% zX9ml^eH$#X>x1Q`7lLKcw*m4?Z-CU$t}j0mC`&#bB)=LOC~-1KUU_+tEIlzm{(b0R zS@z-pd9`JLsjsQh(DPk^d&o36vu@NiR9`^kSN>7b6s=7^EQ1Af{RLOQK#T zQpR_50aDy$kfLh_DH$lzM%>x>Z$qDhES_qRQgrWEB8y0KBQ%c^V;pT`OeC!k(u@+x z4L68)s9rj$SI$JjXBfn63K9=tjSZ9m4x^iRW}#1yqkAtIr1N#XEGMlh`Yj&08dBi2 z3Y$baUk6CeFrA#kosR!S&-L=(FRap+;1@iabiZG%Hw_Dxoa;LIcF1;paUAV> zK__86E9}uxK-?kJd6P&sWila+%hd33rTf^r!QW5%)2fPn#cae^UPZ(MUPzeM(MzHC|3#dP*7= z>m-(EmEFP8OuWCgKP6|5Opvd>{1-X<{REky%62VSwvwLr`v4h%JA!b9M}x#cIZBDw zj*P&66Je$xvw8jp)z`O!Bphk%3KG+SAYXfQ1_VmZV(OR~EJhpttwB;SJy;4S220)? zk)5PbiQUa0oubRZQrsCVd+;kc5G>os%Z&a5W2tgQkkm6Kt8v@GJSRwEh_eVv$a6k& zDcI72WF_*Cl+pV=J2(!}#Y^ z(>GlrHeIls!d=SP%-N-rW2DhJB#5P|8NsoBv2w#Yw8jBau>jvuNI?{WI=TGaUglA^zofk;EqhH}! zC9=vXvRB0;UpvzK1K~f@Ni(`f`R5yC9C@2A80419bH$?aE`dA;UwvpvIvugJv~Vp z_1J%wLB7P_Jj@^$X;0H+oiyBKyQGsZA88xOKNo4fiJLOnOd@^6YZz#dV%i}uiu|#|Jo-zmS;m2Ct`oUM81IQ-@iNce zLoY2R+)F{y8Nhg7kIXadk^6{UnpF>xO!&KD})c#)N=A84;cWLO0A3`D>Pm;m>PYn(`( zAe1n{;bP3 D|n#WHsG>0~TvneS4*ne+?m>L|I%TMN1k=#_X+MabYp86b{!_i$vby@m<+%#LgmXin}D1^xGBhKNYftD z97}sRRr`?tCGu<>!n`{~Dmv-YL#%6rEir}3w&E~Rb~}rBGw_>-Y&c3>(p`i*1`6q? zrN~_JjN!RCkh0vSPpS;E692{cZCgM;U&p?17ZYwFergTE^KCk@M84FiE5r zY3jfRX}7;AzgC?rB~GcuKc03HX9eye%CQ%Hcd1AM?#qN#w*L+KSMV43Psp7|TBKnD zqeUk(=%<1-ktCkwQMRp`$UD@dn6@q=YzgI0<(U_@kw+zScEtjnY^3bg;uwT<} zFY0AA<-e->jI^@R+i3$;$3DUo;%7cdU*n$&-l708Ws)|$qnN-^*BzBnrNMHO^1YM9SUM9VMabft!SXBO zl(2ShJHtFkdUpsjj`aQ)S-OvO=8z63J3dGrIizz8c?!(s><{vTMePZ^@6zw*1Et|I zWv8A0gLGHHZ(%(IfF&_l&JxetVvwr1VChAF6aOp7J78drzJUzJ&urs-fZR4dNJ5m1 zC0)vulR}%Y56IE6KcSs=<5!4uC^?M%flVh?bnnZQhdrK&els4WEs<*eH{PZ1x^z-T zm~E^V#nVOJCVe%=)S6l_nfAgjFN-q@<;lq>F1k^zEzB3g@$=39J=g^CieRXcA+#s` zT*;WW{`#0LtJYFX+BkPyZFZe4vav^P+PWItHiS!Ea`#s@Dp^t+~`oBKuoyn||tKdX~}C*9ZJm`W=^CnpG#OqfcG$k*uR?bPW}=B8Hq z11j@5=Mg?9k@<&kDvvg~9Unk#HLGlmM`WHAx*wTqQ38LPn&Imb$ThM!vXH`0%6P~h~XmjHkk@3WN z8r0g8JC8AoRBKSd3Y}b`t@4n)$WG3b-fx+6b)%(tlTPm7R}#k>6R(pS=%(&5V$vd{ zj66&$B4nE>LJH?ZP=ESu4QoSegqSQ5GKsu$t9ee?Z70XbPjDAHnRLr}z6)rl+z#CN z=*1Coh4j=u%SN4xd42=8T2sTQ(;akU2>o?e#K1gkJ`*A4O=HAN-)TIzO^=Yu_%TvR zeh%X25w;a+#(ygL+DJbyjlLcmAq`Puq=7sud487qj3WL~@HW!l^`qrC_%~3WHAoBo z!ztom7zd-k*h>8&XDsc=o-L1dnS_5qF}B2*D8hz|FYDwKX>YwtyIo+Or#uG2lynn^ zHZNrkF%j0N*oO{-$!{2_HW*ADi_3Mg+r+a@5(0JC()lfRaRHm)x#=2qYfe8jFel9q~`-ko^zZ*wh*_Fv-K(bkHH8v#ER5d z;@r_4=(DTQs-4g4+Hk)^YO<$1yt9SW8@-0!ewxx%yEl)vyo zuq0Dv6YIAT<`ORFHuoRg2dF);kvjx677BWTB#*dxjKy5Szr&ipbr<`GO{{l&xa*Un->Jj zE~J65=8j+)j2uNV{+r?rMt*9c>;dE-NEqBz@xk(Xs2CZZ#v@E?W7x0Qk_bGR_Km`( zSJ21o#g+ZH5U&;r(TlL{Vy8~t>Cj2ZL`~XBZ;I;wx3J%@xF=vds6E&JP?ns##2+a# zo#%y&o1dz1&tVtO2FPum?@iKVCVtykUpl#e@nZiC3ple;40Ru&?twnurOBII)BOU! zyqnbfY2rTJqFodY>+0xsPn*D{8K?Ak8eotBFyAjaXz;8 zO|W?OakeBs9c9@@qjr8Xgt0qBirH5b#c}RoOuPg6PWJUsSk4`w3Hv7e^Y9XU@CDfOq@af#Ixb(VX%BGZLwBlM`DP)Cwlpae%@oz$%nUSPI@Fj1vZKK67He#+pGLr-Ap%*MRC@8ZaQ4ZMr!BsHR7N~0#@5fdv%{!6IWzjsQwO-;$KUgAps$@9t}>8s zl0N>sX)7mVZ354m=~E+hFi{`x)7UfhFm~d;9qQ|+T>8(LNgp&IuOqv#GvZGnuIgti zx{*FtYky9bNEy#}^iVe1tBhx(2&e9r)E(0}^fJcSj>+6B@Jxl7bU&R++|!iK*vKJl z{&XQf;`MN6$FrRJN>E=T54Cf~X=k@5L0{aVldgf?5svnB9jnrbQOtpp=;w~4$~ zqo*o2`Kx=g9LC6U+?}kw3CLtfha&d)dr8B0K5J+H%emmMq+P&1U?>2V?ePuifPW(;Wq2r&M6u>dS-O1FoOc%iX!CljLdhzWmDc26x)MJ$4 zIGh6C`JH{Y+NY{JG&Av4TN~NmsdHNA8phBL(il3B`_#w8`)VlXe8$D2A$4wlaE2KdI;*Ob?|m^Bs7(pR1DGJfc*UUctQq;r)$x$+nM$+MI9N}b#( zbiP9RYsh;o{m1iF*xHU=%q^eI79;jT6?tevX}_ZH#a#?QuBhWteOa1Z)o{9s29 z^AdY~H8+$U;v7%>bEH``);}IEs{Gl5sdqG;jntbnyoo(S8Eun4kb7}tA7O4GPb2Rk zgOOoifIGM=X|uuT!@=8vy|DLC3#z(vQR9CE{wl3}#=#x*XXzUwYrlyFPSsnTiw&fa zzl^(9p5Fi(#aGWlW$*M&-evBn&I_5@x|6ufcgc$|iBeciYdYML@5377}3s- zcd_cM-uV{(-1&EM_wT)gJi`7rHb!#q>e--hmbO zIjzzo4Yl6+>XE`bE&N=>D<#i&8Ea+Tv*kTqfRt129p$s-BG2EaUTQ8Wqpk-D+lQaB z-(%=IZnEb_`tGf|DbEGw1fKV*Fqg%j5BaI^9>QHw{<8E*_~M7TtK~y;hY;gsUKs)8_Aw)I%%;tE?|GRUWH>$coSLqLZHk* zZoNiWQ0qw1c+yBD-g@elL4MC7IhRf)+}ohmq34iZ?%G!?$$Y5hq5|Tnb6N@IEy#OT z79x$@3l!Yt`v4V=wWtW2DMeoZO5dscUnG5OKm{qvf z`Q`}u61c%OAV5AxTE7e6eGO#`>9prmz7<%bmp|h^L)ts2_m}A9W0_~LhhGfioyiC>&UzeM zMXEF=qdUJcNF!%wmByJ@_3|=)D(;zANQ3+*;69(hy#{CRc8#_{o;T_x3xCtcysN&d zmrsvyZ~dC@j4dT9e4bw2e@Genr{C-4{CC{jcnwm1f;&b09On&EL7Ym;@E-fjU*qRK zz&AU{oyaxFRCp7V{dn7B#Lb4?guDH%A<}`~_T3P1y)r~vkVW{{LUYRysYg1wOx*vm zK^z|sk)603o;OGxONz>5a?1O67MdVzMO4Njm}B0r@KEk%r#>CBZ;26>Ni zI+43paMq9I%#V$In8Y0b@*~)ruaiG;wk!NdFK(O2`xeH32mSvX_io!}h__`X>)TA( zTh2Rv#?>Ceesq&Nmj<2Gy``7aUiG3cyhd~mvX`n!Ck<7c%F7~Fz(g*`Z#_M zo{vT!3Xg)yvtS$fV?!#ha_s(nZ0<>(HD01!NmG^gX%+TMy)=GDnN(Rm*2`1({dN(* znP=S3$usyp3-2%vo1eqpXwTx_Zk9YaN(KXw+ghj^5EuvS{ z!?-^h$$boN4;+NUV612DDPny@&t;v;dxg33wE#JeJCFC_#%|Wr$?Q*M32`dF~C_qquq2LAveW zmdC|{?N)>sQMn5in$QxiLUM=gm4eAJEABp?ZQF`0gPv~pN*E*i~FO3xUi%-Zc z!n?TvZan{l-uY#?zUkR9`ab&X4*oqKbKg1h34JefRvV_#v1+v5{oQE2M~3Q415WAQ z{pNpk=U>ptdjq2N_Xux@$ab_=(c_PYhB3?S9HIBwM19nH9{J6 zYh?fSWbR;-rEx*BILlLI5^0}buaharX)pt3gAqGY_esh=V{p%heHOiZ!hN`TPOyB+ zw{w=RQBu)8N-7ULAyvHhtL}-AngtPJ)eV)}o5RIs8!mOaN5y`7gj7upl4_`l4HE0T zAgM*#IJ4Jn4-$Jm_xVQb{BeUUB%ejF1eU^bSP3oMVVp9_-xn}Wpwjoi6Ap=kx*D8}*5g7$VC2$BTSnhQTexxa*lG8t!su ze*k-gMklt}fsG=YiC0N_Rc#E7XE*0rsATy+huU;WC48vmplN!hBc=i=gE)bK6b6NxsWEd!6<#rTxLp-06Y( z9^MbaesDk|IIoE`onftk-XX!#^CUdW96lgemXPLBSPm;80aimYq(a{Z!IF;LsPrPn zbg^DGqi2KL_=MyloA7f&A-XG(zJsmkCE#I?;l|xT-?c+?RFE`*6B@_z9qx2&1sWK) z^#gfd##nYQ4w4RJJGSXc!6r5ZNo!h=v><)wXB+c@g|&zDcf(#VLnYV=SBDMRh*t|% z6ZWu+F<%-aRmgFKDFZ8*4~2^zX+c(i1HC}7fA&{36UT@ZYG;iR8`PoO$Bz*QdNbi% z;DLkCba#w6;jr>QGe+9@Uf>w|aquKXh&w()I*{!L#z@umF;WdRV1?T0q|ZD3I;4H_ z7&%3p({L8fK`&f{E6@im7b2wfT7=`Y6}yG=mpFL zkh6^OwQGz76USLRMw%A$4FGx=42QPK*y(us96Rnny1}FIJ};K~Q??q)XhqiIw(Y@2 z*+iI%)|Juw=n)TZfW5q?Wx zDJ+MTkN~S88UES+Q*m#E%}~hRz2LO~$$Ke4vT^4^UO=GqKFNHC-cyCmAp3SaAtlH% z*a^E~FPNbc9Q&9zkand*Gc;wf{{R=dYZ?2G#Rl=99|RBcgc~}bow=rJv0kdFQw{Y| zb**($w+`xvwA(m~R|U&q;vIwIpuQbHg)}17`-&FqxfR+5(!bC@Kcjn~zKeIf-F!cV zbR^OK%*Re>LT`@en}inlGD3Q1!jMsNnzYZtIk*T{pbu`qEpSH;lsn2VZh=IA#mlrPckv;4S7b1Iq2;id_+AZ9_ ze_4cI)ph0~s0m?i3S^D~+d$?l=0iL4;1a?t1)u#5HOO-Gm5>0dp=B6z;YiBJ-06b$ zo9qj?!*C-#m)U>BGMC0Nr>y)d5a?zH%aqk7+x{YnNm(zyK z|5f4Cf4qN>Vnxx4Siues$Z2Iy-oU*(vgI1%FQ4(Zo$-fk--G=l z-MBr-deUnMq5ib1;{xN4es?09pt%eCw**QvX}Z7zhoNF$pje!Natz&iG*D`82TB1S zeix2rO&*JF%)&;dVk;*Dum_*H;wTN6Yw+NG{_|66v*Q|Jp);Ry;&|8WH!1LJFLm%ynKZAdnQEe zv|Szbw8c?Z=y{pnx)84SQGTED*$|nFJ|7msB3J@TVL7aX1XvAyZ;cZ7_hFKZo(k#U zpwBlVoxA8?^v%j2+s;O|F&130jE^|R#|p*=(zAg6pA#z6c%BP|uoWusv&4l;3A&Yc z_BFhBFGJr6cHZCD?FyxjLZud|zBO0xrEmx2{UP%2B!8rXx-}x5PVy)1W~3u9 zRBn;Z9k>UEIoMDLVIUaYHITG*p)wdf3_NL};>O*9fBUdd$)WvKANt;dCi3kYeKmq` zqrie*Gmm=3^4-OY>P|wW4!3jb<1}pWf z0b49}1UqG{gQ_m--_5hhod3qNCPuL~BI|J5k*UN@hmEiqvY`;RLJ73wGyfJd|L(!Q z%IW_G`XAhzIDbI>3ie+Y*njn~|GLfmdztyS8~f~H{_SAx{}$UKo!&?J9tQc~T)7)P z$Hv{hu&!SXl)bpkPzhG(<6|~E(gDqo|5}!~kRIq=#qSA`J&ig!jO_b3RE{AJGH2#} z&zy~Z3QmKAH76H&7S2I0G~wrD?Y)TZ+Q%Bk`g#Su4?LV1+&0!OXm|2mJ>#$MS=RM9 z<^jf^`!$2yAY8%sjJLJ?ZVUYm+yleUIWI!*3js11*|UxFyCq1%(1*ha7zGvhSx7qq z-O5_1&MV{4CxD%@)JS#?2@Oc;laaf(cbrfSvzaoB|0ebijzidgH)ms~UYZ*CW(+Dx!wPnl-UY%SEy#)ko zpGNtz^m34Ahv69HJ^wZ@Eg6U5^iLq~NAYuCeOpc=3tr6PPL90^`Z?%@i%_+P`X9pn z50F08qT4!2|2pZz6~gqv4Y&nI)&2u{4-9ju{{ieD+Q8LKyLQs9H?jY_wDTVNe-8E! z`@sQ?;Djb{atcg? z8891q7(X${-WT|d7*gHc8llffo6O>y8T=N(aO`mjvZp>xaY!hC}?54 zaX-HjJpoojGNeK}^ga2uY(zSWvt%_xWlTa_%?iT`fc3udT@%@PY#qFeK~NDWxg?F#fdu$6C-T2S+kW9pXv`KRf6w&d|% z!a-Q)Sn}@77T3va|2+Dox|hplUe0Epkj*|JTdFQ(OZClcskxgiR%Gq9Y_UxzkEktT zM>dm=3p{WT4#P1x4yT~y%ob_wAs=u-`vvNDnL6&;A|9xZ-NNs(x5$2QK;z9V;=H{@ znh4VjX3BG#bk4##=!FXB7O^B|%SCi+IrT5zB3ID+z)rd9sIQGSsD&H&sruhS-hrk= zI=P25yg>a=GA@rYE+H8AU~n@&Jq?Ug{5qhjJB#{fQU5INU$Ue&obj(?{Lfmtot`7>oT+#r~lg z+}tmj)3SW;Ak_L_k*}8tq(2EPrPyULb_oT%*DV~vn!Vj`m$mrWU<%>#j_}=)H<0s< zL8jq$BAc!mWCr?dh=H~g&IgI?8?u<6kZw@l1kdH!eDJW&n#Z%xK(`<(CUduez66#+ zjg4|xm>)XG7g+~teq7G8m5>0dAsKqT21!M#d(?DfALpix$jy)qxljmOp=W)zBq;j} zz(#QIRDPT{b|ZUV)$u}?`3!Q#W5eHUk-hkvq3`?_sYE(->|2lpmpF?)6Cif7-0x>{zprt>AI|+gvSl*w2d1+QPh}kr zVci|cx(n`stizyw>wlPhkHK*`1r?dBfAfOmG`h8c^Y^}BDLI-VXK~xFbN(*n{Jo3w zH~!__nX=-e4Y@vQINf$~0WvvV=u*R9wjh=wlhP$Js{sMQkmq*He-Q(iaJtd7oFa@fH1xj@|cMP)v#R|2P1I0%B)h!5Qe;X*%2r~m_ zLk!G?`LGZcLCXr}pCsm=1K2mT-^RYdcTRA!{}{+`OksZy^A9v;GXG>T{~()F@M{Q= zJl^5D143j8=`Dq(lL6wi1jur9S7!igNPw(FPXJFVX*ZC^bn=4L_?d48NHQ`ND#`<7 zBXToXdIFfg0+_#;yQ)H@z&k|hZgQ`2m-ifLyypOQ{_lB?-?l!_yNFl$o)_CRb<;i?Ae~O$DIu+(-Z8naa_&ZVm9iG>V$NU8 z++WNZfOIco{=XI?)ptXr<|gO#6wZa2oC~u!|03p_9XWJReI^gZ68V5IQQC+wFfx&#?oG#J#xUYI7}`Q#!0?Sa0R{KJo}gc&ZjH5 zADB$}rc=IIln>p*?=q_5IRCF8k8;k*HqvyGb_aPN?YR3$;|AP!whdfZrsT0@Gjy%!U}43%>J;M*FZQo{xJWEP_0~Pf+ti z@1tR|6nD@05bS_E`Fh6cqujINw-ORyH6%kS^nDg4>Bx<+8L}Z43c;!cO?}B#5e+T2=eVO&Qhi?;kwinD$30AO!1Dc^Kjs5v1wGZH)A8L1DH&BOe zhk9(?cfS|Qo^Kxadvmy7SU~>RRx`AW<$eO%zyjC#r-bWnayFK)t6TDAF9*0wK8qPuuWwV0s{{404tpn@}jW^XOcF^J0BpkaeMaB?;VU$Xospm*7tD#8Phu1f2OuYvA$1deV-U4 z2Psny^S$Xpuoz()X~aOmXzZ$<_8rgp_YmjbPVTvp?cJPzk#5``Wc@PEzp$UQ9MCw6 z^Y3`hztBv+<~{sglJw@oLRbV99jyPb1l`K>8d!?H9PB-m?Jj!|sD+jIIVfKOay2wv z<31mm3Qo$~ww?8@ly|ncH-bBpeMc7i5EJVgvWjw4Q?44yWrbRF8`N#0{+U6tnK;>y z3x%*1N}vpOLQ5C*@238g%SCzGFHrw$)E}wtO6o(X|1jzg4*VN$Qh&lVA)9fV8@NN^ zd}hK%)i<_FXzz+@>c5Y72BdE#{hFgevKQUGhVwsVteZ+1Rap-ONhSV{>wIgp#(zK1 zw1WL0es*xi(f13Om%xF$8Qf9KOJnK#S&V_{Jj2fedCZY+<7hdE-orijVPvmRhXFs4 zW9Y|0tv9ETr{OG|gI>4@X4cdzNXr?$^dWCR#Yz6}2J#MA4`IV#YsEI4*a-V+b;nqp z5-3&6So=eG&NFqkS7#7=AkXo0uL+lX#5cT%U4ZYq?t$3rF5a)=ZtYvo z&x<@pz$l1-aWDZU!4zm=pWQl;b|$=wc6^xR*k&x}e`GU# zQSfm%-*@m1mbCi#W^x8{HpD3R=lqs-?P%%Y_X%@x_ttYCg;eK*h06T`V`?Pl6!ayq z6!Nb=!`ez7><5Ln>5gW5jYZT+-34 zfxMgMef&oB&0rtM`)?iZsFAgVQ{ViWK%EP-d7cY}3f|Qfyv=%mUIJyX6L!O1FheC& zSt6v`#yfVTm3Qv7NE_}tq+P?0Fm`Z2Gq}J52jMWZgz)}h0#_y4iH|BvGRKmSj_!8ZoSNar}5g4581pA*iayPSN#0O!zq z!Na?Kcm5dZC>k+m3X9Odm1+ZxH_$6b!}AGx$Fa==Y!{OWi+l#se3$ zLkD$tU#I?1zlZTRkNYRaoCDdog0X>Y!rhECEWoZ{FoeNy=;IsAzN=$o1bWZ6;nI7a z_3PO&(vzo?QTRo`IG6wqY-k5}Vw2FPz%)>HrN-Y3bQkv3#u%NA9s?fS?lasGU~lcn zx%kb8g{kGHwoM^|@%r;)uM^P3XnIi*8i zBj2wguP7aEfHRP9?~tdU3H=uG4md1S=FP`;QR0LcBh>0eNeO$Svd@xCbrRb1Qb;2I^bCcI>@Fqy2|beyBgn{U_{au5&;m zx)YiR*PMy{zs1@A1I}Umf1w`!zkp#O_636>42Hu97zGh+ z|9pNfB>q+?0e1#Af$abO#zg-^dp`ZYo&LwR)IEFsGVBlbUsJY*{oTd>kPp7U*-0Ae zot^sTcQ?9YL8$CS7R+S6;rEE1siZ|YI>wSV(nZ?pTOS8w&rCS;$WW<7TEPwusJPAd zztM<^?e;ChmptNIGlpha2C!%FI;Qh2IfL9ze}2r?3u|P80m;J$RgwtSPILbDIEKUmFTX4*#9j4XA$~p=>PujHhZOE z*uM$;$7X8wVE+x+Mi(~Hjcp+9oye*W~Y{&ldvHukMpJ)Bw{}HC(N!n)sb1-S;LLqF05>R%o>fiGk-w5FD zrM-6|cf(#VE8=*EinM|q9N_zYGn_My@*hauW^q13_dx&mf4AxXVc35X>kp_k+6uMk zHmKXfI#nJd>BKn*hv66;hf{DG&O!_4*w)+Z1-5hk&1X#{UI)?*9w+DDYn*=xv;Qt< z6=WlB=Sj}L&&aE)%AQH27_tYbt3^zIA9u%C@3_P=C^HrmT*E?2Slg987>o z(4?dO_)kH1Y1AL4q0ayhedJD~{(;mVX5(j$Nh@lMf`@5GjlF#{X|dPJl@;1uAA4#4?rlpXgTRuo`f$ik8{9)tpub>Klhzs0xpk>haN1 zGdWtU$l9oAu_5bl+mQz;Q_f9(9{vST{ZQBTChP$;COLR z&&@o`hNg+pl8Y<^RnE37>NPJ~w&E@UH|6qlkT><}K<>?&DSt?uC3~*Vl#doaC%czT zmk$?DmtEJNl@CIu%Fe{8^8SHmrF_|B*%3Qg%6guW_s&d|(q$9n-RnED0YuuTiU=_ibtJ>DnbQL>L8mVTTTDW4wVTirPUQlT3l zl|2Jx3H`P2CckIA%)LQ3@6-6rm+jCnsXN0v!&yV6Dsg~R>xM|p8Q%8{43T%H{zSI! zn)6-)4`I^ic`aH#D#jKhTHZ_gnUrmsAy)kR9*>syV`t0G))?N|&X)2O z|0>mcqNJ)gO02C>Qe%svPokuDUle!fQM})b68EJ5u@lz;&ENtL9E8Krx-3ds7Dw^! zB}&>7qj>ic#lA2~JhYJ;*>UO}-)qdpYAxh0N&VikZ(Fo~{r@3a!g$_0I$A~`)&1Wn z<$jWV|4oBLpcm3k1#1Ij9D2nv?0>OACZJn6JQh4zF;c|2r=V|r*;Y#!b@NlMfm*z^^+T$6 z3w1YH|2fAQ*(aIU1LhoJPq&Km6#rAPQ231sX>Em;lkES_u>b2}{|D`*?EhG|+~w^5 zidp~J^EG6$|IcF2j%>v3WN+RC&ER0Y&Ly1^D1)8Ql*Rgw|88_w18GnJ3QogWI0wCO5!AQu>Nlu;{4PFclgJhP`XKKJ`&-8MEp#Wc3H9%Ut zUkhWu0~^@KGw_r%7KhQ!^soAs(Wc64p)UAU&6_3FX|v=G>D>dv%h(AFhAi_4X{0#=WRa?MJPfMim-$wEMW;tSjLN4!VrexVk|I1iia?SU_uCHFlkFj zYPDLT)%w*^cPQdwD8|LG48^4gMN|rxP}r+bgd!Bd?B_IwIFrnsKlYDrJGz!X zobp7+O*xAF3bdVZMv{P!yMZ-V_x)_1Xs$p+yhx%H}j20yS*Ah|+%Yq1_1u^DaW z+5eZ>|77Mm`~NQcA6a@1jhET~FWKX8CR=}LacCyfbJ_o7t8`PN+2+#Oj!NuAi@5Y; z>ksIe9nv1k{-N(j?hR>ga}Kg|PjNUXF1DP_97$BukCJgzt&^{%@^!p8B!yFGc$fYj z```5@$eOe4|BH^B%Kwh~vC0J+l;LRoZ>#@gki!Y&{Q>NhXbtd5;Wo##JNEbfH~x2J zB#Rsxm9wVXANk+ZAObk2B~)M%ul@=%Jezm3y9VLKU)8racvzVy<0=Q zW3Df^$?*fV@#>IEj=iOfTp@cjz0g3b^!xWj6sO@`qJ$S zTQNj<7)nt7wsz+o_7Xk6KH(Sg|E~O>mw&{s%Kv@&kID$5vNFc8<1hh}Fa^_4iWz8A zp4*l2j;rebUCQRZkG228jmlZ{U8ZA5q4~D*e_#10Tm3J!)EI(v=3pKcAo9(pH<

Iabt*H@3c-%mZTQQVB^zP{&O z#Ae|#l>bb79DWnF(?5RyDus7qH?nNw{p3L$Mhq3duVlN^@s{)4!96@c%SiRG`0}rg2pQ!&THDZX zfp-i!Wjiab3!P2&5u58hL+@K`w58VT{M!BayAk?6BSNv`2Vn??p#;&`+$gfgy&OZ1 z!vsvi6ih=YW}sI;zx;jU|KIkG2={!I&y$>o1z3b=%ybF43@fk-(b|bB@vG^N)^D$+ zXKRLsf4hFWW}@}ys6!m}ON~p!q)pGVZHE1<%(>*_b%lC7dO5i><$(ZvA~<>+g%KzbBhV7+W`gpoQLgUtIe$VYBqgupO0X z@!zzu@tyRH@$!zl&xGCdy~z13d(k=0Iv2TLTueF#$-}7LW;~TVimE;G)g@m$Ae=roW>cnU6=pcuJOM7lbs*9 zHs{SEhsF`|e^vg)r^sfVb2;RS{^fRHyaW_y+-|KzurN`x^&h;cV$>X|*d21l^ejNV!)Hdb7 zacS}nc@Hi0?;3A?rEeJJxG@-q37CW_ zn1=T2-ob8V`>rxhb_sWqIdwsnY#M9+&uD$`>H6R!m23UqW_{n*i)@sE)*VV?24*3) zL;r%TBC9cvz6eWDC*Al}_0N6vk8~T5JkMUAt*u$8KZuw*A^W33{uuoNEc3q#`|edD zzrIl2T|ZDips)6&NS#csaLg*KM(@vk8;V?yjZyf!zDJ;*?0#Pv5$+lOOeiBG`+s{B z{(|~T-gbQRe}qcmo!E^Q^?Dk6>6!GyzUhAYK}0rbcANHtjMj=T5O)|c97P;Sq|l0* z0`t02x4}Gbws*a@IH65$sMS9}qqe?@4R-9V_Wic@{ebozEpKSwx3R;>c66-KKU=BK zu|?m5%nIkoXl!Nc0lzs{G)FZ@p1?_*Mom%QP`kzbn`@5P5$$u8_E{e#KR%(YZrs6+ z+rzG_W!LG$H2Xd6_ZG5slRZ*qvjfqwn13Ff^e#NwpD1d7&p1!Tv(~yjr|(T~IV25S zpl9CjU+G?=UqSA^|02Kt!7etNxY$sA81e?HFB*R!Z=s4_cUQmSJ$)JBduV89@1p*c ze7wv4)Q7A^-7foyAF>}Dndq`kH*Fu(%gUbr^jWBkVE~FT2tzOoZN`7w(ZLqapp)K( z>}6%Z``K8l|BGYNOi8oZy-uTLvHmZz&l!J{&M1t*I84A4OhdlEIobO?-F{QT72jt^ z{>r=^`Yc3qZs(BmumFpY-+x2=5_)F5a*1X1708WHE)mVa?bJ5M-_y2ZmETq)nj7DM z#BpWq6#rC}xjW+5Vv`kvo17nYyNb2b?D9kGa`t@#J=tCy<~YYZEWjcx!7{ACDzxzdv>)Ij zIKuyc&iQ-;{0!OI{10ek<2SMOkMU8Y$Y$Ym6aNER5v{*pEuFPkkBw+?o$0|(h0XNL zKzZxWM?o)tn@zPr-h{gx(@DOrtmW6=D@)P$cdBRlUAw*A@m20$-CXx|q4oF1CZqX7 z4Z=zHq@GMTw$gDsu^W4_9|tkXeLqarI7cn&hDrzZ^u%({2a$i`v46okmAcKo@ch%B z>!bN4f6_m7Q@KYpKQQJTM-fL7DMaJ@tz=}+M|=83^8+%%Ih?>roW>cXZWe`e9q zN59|={ftG*1d`h92H}LhNwfxVtGX@Phv)UV)-?IO`&*wfR;;h_#DEa3zbh8kYv01& z&wM%zqE~#?zHndm?GpMhZ8 z87DLSm1%W(et$N4uE6tA4>YMGj*%(HHNWinEcJXAvj67#Mxb;`F$1$O2lLSTJe%UD zPlaggzuP9bJtkT9y!BLA;J4SKc5{nzk43^uko}qVcc%5a^fcKrmoI}}@kROlzBNqV zi;Oy=^S%eqJ4&TidvTT#FX%@_KS3 zGW)cn=vv4}fE=>e?6)%f(fGnP?PS`Ra*chu{a%UK=)PemSw&WlH$RQO7yEG#hY`b3 z#F0b_t;isUSZ&|%le2~4$VKZMSNeYO$Pr=t^}cw_}+V;47)}Q3_n<26!vZ?vJYhl7ySOicl%q5SQy^i^`!Bw z{^7?jS^tNMe|iwi?Z|%{ck3p5u#PMYud6SPysQ22zgWTV!cQ);=Lg>p@1*aCqc-iV zF8Hmo|JzWr@2>vwZ9dxDA<=Y`ul(2E&0q7;-V9CMAB4t&_tmYx3dioh@7=o2$8bGd zk!BCB;Rbqf3wLl2576i9-v3>Np>|(ks6)J}Fw}PyhJ<_5aJMidZx-r*|1R`%+yE3~ z5QbnFO3=2cuk{~&t^ep7GGwPQ$1XA}ocqAu4WHFB-{1xcNzatUS3yT z7D#gumS7oHpv|%Ej_n{b34NNhzD<`t4w-Y_Mtzv3^ZXM?`Mp{Eqxk`?!mFgQ8nLtb zIb_xCK4Cq%5!EAn=Y%XnoG+uUuWMW98vDC8vSAZ@>)C!`yWe|$shpGD&)SPB3R^3( zo7{{2=>59=dC0?vMd5$4c7&{%d!K*heyBTpKg3ag^S*Jh`=J3z%y3M!Hfq~Y_08D_ zAugOm`FG?IZ5uvR|9zm~NN>6yGW5*C`=J9l`U&I?NPCZS%)IXz-VZ0m#YWr@ zr^z#@o_s%?BQKz8x_k|kFI*D7f`;=ShNNq(FOW~vy!Rl~%2!>Kw~^wJKm(HK@tkJkUi>3RF)fHF#t)-2Yk4^s42QO)sHNHyU-ZngK$B(r(9b>UWs&ELrR-*gWU1YLqjil3(?qe`Yzjoo~hC|am+pX z1LO{Q56~sPGok-)%(b?NojUO0xNH5y$p6j09;^KwZ7)0jdHt02f5-cUt;Wmy{BPb< zj&DcAD~A*`Dp*268b14{V&>kAR#{5i(rhnlV~W|C`}IvoHtqumFp&1k2EN z{ZHh74T_NV=B|Qy)H~dNS{SnyvrK{)pE1Y<5h3eGk2iUhzfs>bLj@#_Io#_ROby_9MLm zi1r`d?wCrH7b-8|$3I#={~6)5#RfLJ`Vlo-SqB%@PC%pGn%C~kosq9C4X#9u%KlW?R|4-l~PU8&D;R62u`v3Clf6lu$ zJYN5!@17t38LzxK#}!<|4fNs`?%*EU+{bqOQU6+hJ9kz8Uq8QTss2Bil3w#_2Gz-vHqr-wrM&Bmdu^DEy>%O8q&6 zUil^a3w*^~1$xUV=eW#vp=Xr+j~qx96KJSqpQG_2`}za+^j-PqowNE4>)-hh=FsB@*uHzL5uh(X@;=-5CL8!JTX@@3VUf7#ycNH) zPjRO@-L)(cw+!j4`~sWUu5-;@5MG7sWPX9s`~o=XdUU{*6j;JFy#ku^$I<7%?1097&{*9}hq8d0*y#6RvrQ|8G9u z4B{*KX14Ir?9ommiHu`%$hY6x@Cxl!Wx2Tr;!dN@G41Gxjzwpa`-?21yfBOt24-OnBHv3?2hXEtl=EorzykUr|Bby0e(7DT<;z3l(@X609v$!=q0%usQLQf6P3}dsCgk<|#@xxR z4$3^o4@Mz!u>>)mYeg5P3xbr2ELMx)RdFk_xr)Ng`um9)hCy*PfP88Qg zc22f8v$&Wx|1^09)%0`Z1yt>E-Mb!M_a)&g$X~a-)UT5dvSzUIx6OS+9Q8=Nq5f}j zU(w?~*Kh;9xP?2ohX-iW?{4RJ?I1J!F`ZuY5w;|DL_T=_izlVPpwXv(@e77>vUNw1`V% z5Hgn#|983n``rI+?mu!}?tlNt2GI9+T&i3B zhcsHQs{bzfPO-FhV=q$btNmp9GXI(Wz(M+9#Lyux(OTwC?RVEy_sxCHp?a6~H-3+! zYKMMJbUd3VMYf`0o%aa!YrI#>qw=YrfsEgBIDzhR`z???pEoxm3cKfLqA(jH`>PS* z9Q^`%`H;(hXKsNx2A87nm-~e)(Q&wj8`$>Id(=yh)ZG2 zv!lBtl+tG)cS_pgy2N!l&n$7V<0WAZIS}qs^lwC5|#)rLjygzOFnDm z({DA`39W&gXg`hORdv2Oiq{L%d1 zxyt`S<$tO2Pv+Nu-_-v@e*Uj9_|?)|i}l!umY2;DSZ>Uhp4lVqH}vJ{+mWCDE8Mls zxyZ^W-ZP3#HD7{WHP&1Saxbb!njb+PMBFvj9rq5N@-C9eE$TS(u-{@h8XbcqQfNg6 zIh?>roW>bkz$HX|hVEY)qc-N>^Q)2&t-FtWo7)hLi(GNcqxFA1^lP|*-iXh6x5zuV zhX+xdbowmwZejq65zYS}L=M3)lpsHchU_s8nNm-Tp^w7^OhSveXb+$%^!)haX!{J) zOOexO$>P!YW6fkX$Ts~H{HgiBTkrpV{`V~Zui#6sv`^3+dil2tL)&c6A03N5e{|Bj zklmzyKz{yjt+@wXt~G7$0lO!S7QaRFf9FYO0Ty8imSF`}q4!^&$*+It{)Tr%xX0Y{ zwd8ti#AZbEf6K`2sKib@n*Y0-9v?4nd@FnD`;i>X?m}WDJ8Uf93){Dr?OV5y{kxL= zyM=8@=GVzB*CzPSK^#U5`TnbZW3>P6QQ^G(%LeYa`;q<6?(I7CG5fch{flEr-Dm%@ zdDCp)XrKJni{dUjuXK_~p%u|@>D$tyXGTjK75ZV7e1tiEiR?({AUj=q&M}b>;RJaS z)koB!Ea@DV&sf1dB)Gxp^4ze$*ao`13T2ia|XK`A){z0W!Bm!rchdbAgJV*W#4 zUvj1Qu|Rq3@10%jd<*p{-Y*Js95)XOko|2@SVS(tGUUG_pudn^&gX}=5BTHR6_55C zi@t@LKF=i8-*hAFT{9&7;JqPX-`PRo&6huAkIGMlAHFvzY+v|G z-);Y8*ek78RPGuPD)xLP?3nr)dteR?I|mN;jlSU_|ZIH)?pP`cUAC(P(R{@kR1C3V>Dk- zw>)qB>v`qq*>LRs^ZdKk)SZ3STr=erH_(e)xPz#V9nCp<{UV>JaLqve4StO}a|YrI z`8W6q66^RkHt=t(DGm=D({H)w_+9nSC+#^!FUBAYL7Q>>_W9-rAcM}8<_4@WZcgS# znEx<>f5iHOV@RR-UH*~x_(#arP3ArY|1`AIk*#B;@#B zvITq>gZVHpMOn1W{|T`wN1Vb$=6HfQ3%gN!%X>-$NKs5iJEQpe?%SP zs7C?~O}_ z<~=_)Q~ReKYIbaTg!XSR-(Zn3YU!;;s-P&WCDY_avRs|hVtr!vxx%nnIHP@z=HYhT zbq@XK{5*o2+CIN+Me8(8#RrSfc+Pur^8`qZqzK`pz zaZR<8)i)HKn8bhI{mQm$`RodjZk={2$usyT)H{8-Lwr{1sc(Gu!YwJ3F<@UgnO8=FUam z5E?))#vpXeWzWxKvoB@4zr=<|4nzDl41J_uLXN^1jKc)v`zPASx|g+MsMnt5`zKL5 z&WF%=^`rTRWb_??=IeY1XjyFjA^$`hJHH(r$RL_~(B&GUxd+ksMC>>pi1SRrG?b#6 z|Do!v{eS53{?6_Ev*>e>)aEuIfqHH9$M-**=lAT7*z8z9UxbQaJ^jzj-={Cb3ar9v ztVNCXx^|6o$j_hhKdg7mMr=kIwxbd|u^Vk{igva|e*c+M>|ZwRAMAhD#Qtb!f4sr| zK(l^H8ZF+>R_|xZ82w)9?8iYIMvL}0jTrsW`*V~YM^vV>h}y7DWWVXST6^{SErn`j zy_L+Mie5KSx$@5Cgv)=S-=I81`|8&ZRSuE;?hE0h-yVI>Gg_0H&|hyDuYW=2+q?Vf z>*LCuzRKw+U9uWy=oQ~|f4^oAAo>MdLQjn{L3Y2xe?#_uLs=kiL^^Ka4({OrdcJ2| zYlY)qC=C6`0Vu{Glz)zYog9V|j6&}#{lXY>98%-;QNH9G^7JW~h8A&Yl+rV<1Nu z27MNC!dc-ivhyszVzmFyPtEPI{x919=ka`|v-awD{5&i`_MI2PB610OhZ`#}F0+iD z&wt_cu!5fQTvw6lp`LGlYeDF1u^t@->=SgN3t2?-lWKOn7;3F)uS5LZ7uAn1h6LGg z{Kb&G|6X;Mme{AZDVZXE{q5383$;rd0n(w-1yDr3pkD`H|L_HG7kAKbgejv4{ zG{hZ~w!XY&W@$*$E54}RF4P|s&Mb9(b6wwb*EiPnjVui{bLD@b^B}%bx?80ChWxiX z4^q--MFu&Xz)76O8MG}|{?LI8I?;tJa%g-{{?E%lqP-59?M0BTm4El56*2ewoOCYW z60V@S|BLotd@=OU;{#or@HP4kB-gk$*O^%7S_@tbz2a`+4({Or`g}{D1q1NdU)?u^ zV&QC?`Uykm!|(^=U&>tlWaUp8Y#@`uC5{<|F&Kvln1m^qhPLba+vs>h{}!G4zFo-D zb7)-1|F)F>jZ9rP_I_7?94++LZgJ|6Qt4DYtDXL5`|;EB`*(Hq3A5-KX?5%|$Cy43 z(YM30=tAc<`~HWgLrgugz;BCCE$t=bGE~v)&hsl?w*QatDnxT;lBj=IKFOZ1y%@TG zUTTkB_xlBVM>r-bUu((r*odfnZYHDg&)#po7|Q6C=>1aZ!~F;OdtcweHy6zF`n7z# zW&V@BgLgV+H}+yb4&pFkIEpyh>`Bm$jsx~E*kkX4c6%5gd)6KXXzc6#Df0f1sS(~E zGA-OfwmPonkoPy?9jWsEc6lergm43y++zQLdVK;UohyY_WRSxN)M(FZQ8!lmgJ?`S z+FLe1{x7`UF(+{vXK)S|a0yqCw}1DrfAQG<)y_X^-yiS)`vF_`KKqwUU)Dc3|B?O6 zUhk34HQYciTG}1cWX?Q2qc72cTl7K54P_I#uDt#4I-_zQyQ-|2Vcy{?UvyTF{E<+kvB`GX~=@0WEhObJLzb^vq@X#}xWB&R&hrnKX{GvMGtg(V<{d=dfzeU}KXpM3GaqAf! za}b9SLvJJB#Lvv>p~unls`v9vzKu5PaDL8jDlUaqWRSxNlz&OP{`I1e_m54MpXtVE z#h=C*&|g9{!(>*FWe-4nQ#mA)50& zgdBzvj6$@xaJ0wV7+48=|p=QsK76#`lpQXWzsoeVJ{F>`>$EXsj~-zt;SJX{DOX|6j+p zMYKPEE2@mGXTPhimsT}P`} zYv-R-_tT@j1=EX*!!mkigKNVI`YPnuzgcuSrgJ6xcb)vB4sq1)F}DB>NMg1Bti^h4 z#AcLXJ1WuUKDF;+|C5y-tJ{5LKXE*j@ zKdJ|se=zi^aF8CY$*99&dJM_Ft_=ytM0*(K-?zb@#|Zw|==kUul)tGDf{HKtF3c~D zb&1O$hZ8u7)2Mla{kuo`->0A6u3vs$znq=ifaGm+0~~V>7jOyDTKp?y53Zq2Icisq zI*|YV2HAB=`MjxoqH!1dcc%7#uC|_RUaI{kTZCK5X#UC#>Ga|j?x1C+{su2&&W#bw!cg-zN+cEd1LNY#OK+R~3mntUUMv5RF^zwiLym`Q6C920-|2y&%VJH33I`7@|{m8HTIb|H^ zxN#t|=DPMD(KlY>sHZ0o%`uDi>K)}jufO+ju0S+rV6X5&RD9;){D8Kz{NER~QQEN# zI`?Rs4r!zI@y)+s{@ZNrAO7I`yEnCeXhmZ;`yI#nv)|D?l>LqtdMon#3ly>6#bwYb zt_zR$_dV=dQr?Xic@%LZqd2sp=BD`z?D;x&e*Cg=|5G1Zzj(ws$GUFE%O8M z3Do#+EgQXVw)qF%iF&qtf^6`PB*|O;^YQrscZBcZ0kU5!2yNSpe;fDi*kSzp0Q>(9 zb$gdOK5eW-pR{p0`@h@#AEb~U|GsYgn|;}eOkd^eqWSU9Me84x*&^jj+@twD(KmDY z{k1j+<$fV`q|{JxnlSx-YBh6Z|aw(>P!`9g`f=-Zs5$T6rsVqAfofM|YD zZLK{Rj#&HXdh1+w;<7c6NZ!;}yDuNl4G)tXI|b8FiW!)NIcVR)2Z?C!?G8kHZ+9X` zkG^%*w9h?oPf~N;1A2O?dw^Eq7R-~z0xZH3EW-+{!fJfnep)NM5t~u@TlehM5&8v% zA^T6meZS|=LYesO(Qnv^-Pnu$IEcfD;V7bg5aVPLDYQm$|0Ew|4kx0pG*6Pz`oPm< zwEy3kD6C#T7lps|XAk#ZjoRa}@_66A-{K$5FS$g&g7Q~Jct6xX2i%)C+@tyK4RVN0 zd?u`q(!w>|K=oq#|IfE(n;w0WzHYX4;!h~=uXwhHo(a(!`UL7{J`-*^Ce^k z^9v?RXS#IA&Vl|n!W;l;H3DFe?R{;vxa{P+oe^B$NTeN=AS~eZhoiu-PntA?_$d) z<$s;AE8#kAO|-|(aIPjOD^1TV6eBfV(9rV8+`KhpT?59Gj<1_e&J)aJ{M|?VbZ{VlGu7RHpKUnwK zu=na`L(Xr#Z~krg;Z1WMu09jqoc-CbZ|F1O$I-Q^!%q7BG|r&=bDs)t-PFE+?$hD$ z$dd4n$4kPY1H-~wqK(A>l_&L&E+y28AEi4hsA5=7zrvd#6sY{%u0o z(=;J`fAYAnJMkBx=kH3w|6^RB`y2M;SH{jc$JU$r70A}~3m1eh;R>SX-}_ui=%HUj z_dE7$FsRV;s_!dwn#(V)7q?LU78{t%zQU#@AE3|w<@ZnW{}nzH`q2lV7=thb!%%`z z7=x|T_1F8WAGCYpgkyJ&W9pY3nb;@1{hsnSdtGQMcr_gB|7u8$+!C4#{w|~k|6N$% zTqiKWf3nZ&=RB!@{XO4T6rO@?q5Afd;ix?+Yew2{ z^4%x+(7zMn{lCL!^_`Fyu^}{U`Zhnzw?oZU{g0daABd0F|Cp@*F~4|{eH^x zLu79q_Wz1+8&6h`9Hqz6wnhH?%72mklbs{vpN!@RES3L- z@;_IcHGWCyq|k~CTDDlDhg=kHvbJw=-*AF{61f9>Tl=i>v*&+jZQpQOTr6D}&XDI& zJ+*JRKwd%>y>5i(db@A9BHV+9v!3f^&)K*`B6{}i#^Ce=;-~c0$%Hzg9*_3_xaL1M z(2HBRgL`;@KC3^P|Ifyce2@A4KMp@A4nKT@>=;XA}AFcQ_A99247QPbqQ?s&CtClbnFa z2NC%m;+P~n1^Mp^Y>~&~`bXHR-?mNv?RB#EU-TXI3wnN27`mT`_WdXfy?>>?_*|iV ztDX$g{J#|4{IrC(@gg1OYBoL)6;X=r|a0KXxqlVMF%qIL>IEiVUV@!=7 zx;53|5S^s+GqYfJ>Jd!H3z?pz8%Ty{wpqV zTUw|Tm)c?ee`Nm;_e|+K#qCCV58Hp2`Tyqn?-kyU>_mPEJevPs!;a4He=yti&6MU+ zX}{zgNN!+*`_Ey-a1?PQkwPolj<`SQC~|-My1ygb-vaj+`SssL?(Zr0_oDlI-u-QI zf6-Fw{`z0+xce%d98TaQs>M|~_B1{2|8@RMrG+wICj&wqcD-pI~xTFCxis;rUu{kd*FEPJhFrpbS6l_P0o z%h?};*&o8u9-*CNO5b6W-^O4ZCZJ`X@~JFOqSt+(eBCzJk3J0z!pZsU4>EDp+-7kx zeU2IAEL5L0mO#!!)j;D0Mjncnhne04j$5UsGms2#YOW5l4Q@*PuamuKjhUXLn;b?_0K~qnL$<^()$JV z7ySgvpBJt-KV_=+0eSy)t@c6N`S|<-UHgKLQ`(nzwJ(>oFUPen$W8a%J!yU1 zK0PUX8fS1TdcU+6wWyoT4~_c8+BZC!hgOr&{-w2lUD`jgUO2%{Z_r;!vf0l`=K?DD z2%~R`U7}w>53ZqYjq&}y`~wC21KQ?J?Q+yMXU7`fN8>jB1sv1HrdIMVXn)ho`4z zKdk?b3dfN|#XreU(C*VS$l(M|;xx|S94_Dzt|0mjK>7QxhRELT5$^dypKy)55&ec+ zNWE;GEqM1KF@%kUnJ@UT~5;p|H5Y+=EWEEMRP)B_2 z)ljk3b-RYT54;~&y&v7)kL&D1bw?6o9N*1OC@(Av(fWa&fBmyCE{gj?VVFSnzG-fb zcOm*lS~OQ>lDH|D7X9|V^1bV|(7x|AeVo@qrs_35t=H5~uc@D2Q$M{H8dts+n%2A) zj%|7^q}II_nis#uxBgmaaa?QqwNSJ0wNShKwNQsR>XASLk|=eal>0Ixf<6kfFbDHc zBh6aWA&&YzW#0cX{`WHe_p-3SF^jMS%di5guo`R8wxdk{zfAwXjQ_oi|GkX=tBn7v zEaaA!dH>7!-^=*l%lO~R_}|O;-^=*l%lO~L-7O32rLz&6QHGX*@;LOhu$`Wn?B1Y~ zz7x5=?hU#e(>eOJ{I^SD&iCj$bk*+1ZpUT6Z!eFfzt7iRar@EmlKX=EeR6CKy>`Ci z=PS_vqw5BunMd3X#UVzdemp{z()FJ{M+^WAOHTfHY}7m zruY4_(A(+#rFVZ)z3{L2@2{~Z%0gun_m$VePO|6ouZ2G3dc@g^b;!?aBa_<2=vy%r z>deY#)zdrdx8t}faxb|b)%585JqPKB5yMeD`hHKGp6SweK$4z9G$xSUru`&4_i6ve z@_(als{R2Ue}8wW|FBP6qco!b;RH_NG|u20+OBK=(Q#M%k4}0QvP-r9+V{u(n*z4( z2>ly*OS6rmZymN?H2<$j{W$)k{@)0D_(-$5*8IOC{lX=B{Jy>kuF%Wh@lKr5H#x44 zLe`&mo`3cX<)g>%*U(a{KSA~)+J`W|E=2ub!M9uar1ydzts9Hh-gV(|{uj$X+dTjM zzcucCTKTUv{$mX2mbC8R9v-02xAn&{0RQ3qql@0ZBi=ux(2Qu0@s>T_Kg8JM#nKss zAsB{gwt1B?pb~m~rLv*?j-ro2Qh8|*m)M}JV4S!Kn1m^qhEmMHEIjr<@lnqao`*%q zzF@4DtO)Gtr|l0&kLKgAAk)*?2b1lMLtl;D_x@b@Vk@9izvc1xMnZqX?>*ls2qVZ20b6jer`@hBgCtJ|E(Ek7CVs(F&&EFjP0Q)_h z|JPstMg5m?jvP+nG|u4v)gGK9d&%B!DWBhT|9<&oh^dn=h_Bl8iExR$g6ehZUGf^D zwG?$o+|`G;>D{`n4?&*r+XCD`FK*!u?%@Gy7P5~Qvq9OaapT(c3AX1xHmGsy(zZQ|Mwv5);Hvco%#%Xi_?{(PzaPlZ0~yw4bbVzf+VBlqQ7pl3Gt zFNV;EA-9n40bMH}&b2QQ7n|!lh2$7iuX`$tBPXEhCD%RCb))QDIch* zl0SK>BjfGjFN;4ee~9)P*!sakKkn=9)g)==*WWi=PfMSMwo{%zqCHkJXFsy<+~e#4 z&wroizr+3U>>uy{=DDYZ^UvRLsqs&RQt8aVEX+a6OY$#|^XM66wL^FTeGziPS!KA3 z>})cxSX}I7>&M9zsP6kzSVgWzm3-C-$K_$I@Om^{_3VY~Z|g^rzh8gPj^@v*KcsI> zHXxbSZ}*?gD8qJCVkdTEFWU4g|7ZIFrt&-|^v zVGMm7CSVeJUwta{)_8w5co&7Ip%gRFGSNFWxFF1;XGVB`=yT}vkn3g}-dBgB6NAJp zz#=R`%=IoKS6~%tc4?orZ*_;XPgk{1_w^CZYM+oqV-x>DJO9CKwlta-v#-tnYeDpV z-N)bmnW}w8{`)^;wVOk=&xrQ_U+p|=u^t;yz0UqO3w>9P9@h_u)+UzGvu}GRwaX1% z56AH9$*uFX-HW|H-oupkeg_9gdW-XKFdxEyD#CE}qkAyk`A0kdbo~P|i^!K>vq%0= zhdAoxC4mO*coLP;*oocPi=T|tW=<{$Z}V**8CVz|um9Aie*FCx^}r+lPGgn(qaSdL z%pm6g)1L~-_P-1b1eTYQJle`80U z2|uYdF5C3k@Xqa$aP;c1P)#rYXEyp*hKDnbIfo0lge&MlQ^B*^;J>ss)P9P^?3Z|hHGcXHt5Pjco9@)eHzaR?p(JqR@^~Qhn10vse zbbYV-s0Y;^=H=7qcw*eSWn-GT)TJk;zJ)~ z=WXxb*b(8DyshxtW|X1&z=%+_$G({Kxb=W_sHE>ia+C3Izvt)o9fkFR_t)FHqA?|0hHHS)h){x`@!&N(J^;F)lNyo46@z!kCw>858w z$87Zit_j~jcDjCnI;3-|XP_RbnfXkpjpkS&I#>M`X&=xxd&9Z5>HpVz?*4ZRcW@66 z(C0heMYJXKKlCX&);JHD|NbV2OGQ|QxBipT5oZ)$(}sq?u^^8kjwD*zoukP*Ec&DUm-F9eGav4JYfK;ud@Gd+iQ*fX#L+H`Vjol`aeGQ{QL^_$1uN_AX@(y z`Ts`ID}KuE*Z&wpABPE;gejPY{QAFL{QrmeclYU+tDh5SP-c_ImH)TRvv%AJ%)%VZ z!vZWq+a~WHI^JV1zN=4nS>KM#BAV;ns19kuvDwN$9{Zo@tx?=y-;a^bGOWNVv`qE> zOtg;=Ju_4LfwlDY$Q|PULl-)+<5g>zv^5+3wi(r1`18r_sM?@h6)0Dz6yAx3i^>)1 zkwEl9Z$mV{VYlC-JqY)cl{NbJo%W-qA4UvE5l0d!v?7BXPT(YZzNro;qxSa<8TkUw zkr!|YSI~oNxPh9Nm4DQ2a-FEBC-A8JXX~E~y^gzuJ9zA$tmU5+ZkzA^FLwWzEC1*m zs{G@z{r{)R|2pL#t?2n#G+t+{|L^#aI(=_in&qFdmtPyZ{)K4$b8+bRx86$>Bbxg& zh#V5>-?2v~S%OiB<{ytCAI(1=N1uR6h@N3)sb@xZ=?`SdX#VkD=d0Rcy|g{7`#5e2 zrlAxuYxC>(Sr2~5dT=s1*8F_&vrs1--(}9hRr~Y_&%**NLW{Wc?We*LdgciG-Prds z`U>Q#*zfG@t|s<7Su>dZJ(``$4vv%c)7jr-gK(0}+lL?Ze^&YbYOF=Mv6%H_n`>%E z$5j3Wbke(!y>FcVw)F=~+27JRCcWregU!-Uk2L;HZyjphZAVK?fp}}JCH#q zqHnuIdpzdw`1=d}aRSV|D$03FDn^St^`9hfC^ z>cqx0|ARD-ZQ~?!X7|1p6Ob?|;;u z==aZY{5&ke5-daSubv7k$W@X4Ezfj~b-wic`u`{UhPCt-agmQ{Jw5Y*v0-eaZ${32 z$*OO=_yIb}eEY+`%zt;aOS{fa&3|8qOb+cA7WhvYwxbd|u^W4_A8qXHcI|CPk@mN* z_IHH#m&_tJng3lI+k|6m@|0tn*^6oO-&)X$*m&ay(m9M6j-pyz^j*6+J^JQQ9g_4E zlKa^Iwd{X>fO_)d^+T)QGsvO*Yu>SMgm8j>5*5GF4u6HuhkgdRlQ&H z+KXrob|+agTK`90>&VCBAM@Gxv)zAvqpjZINBxU&{yzcb&q>1^z$x^$`^vvGAFnT= z=l7q|w&eF8>d*gxW6FB!UG2YqX_}0_5!Fgoy!}Fm_FbPQt(ZA|rDRpY_Z7_1n?aw2 z?6dY>;bVv+dS??Oyu)O&$U2vo%nxwPJVbrY1!U@o`7`7aWM?Y7bCt^t%01b^_m)}1 z-|x2-ScTQ-QI^+|>#-4=QHC1rRPDl#_0P3k31zyWN&Uc&*r={(x~zPYsSlLz+sgNS z<(q63Zd1qP=O;NfqkhSMf0)c7`u=eA{p{_|Q;D6}jePmNYR&=uQTg3ZKZxW|c^~K+ z$oi4`FE4vO$j{I8`_bq>(Q$~@5tqNhcloA!FWh4csFlp1;_L4FSFQ1-pTJ3!mroC; z$ul^IUSk8Rk)F?l7@aSf@n<_wd)xP?22#&)9pzwgmAUEU2mp!eD2 z{o5c-Jo)-U6jv{PulY5g)B-p(_A_g>eI@fnsZ67 zKKr8mzh4X`^!R<(hEen}NKSTbi1sL`$2f7RL#1H?ISDOQrC|y=4e3N_=yT+%zb<8 zKUim-xVY+(u79j`;`F$@*2!o2=iPTCkGp>NE79%xufG^-aKKiXex=G~6 z|I6JQY3;;r?8SZ@MB5hkzrg)R2A$}#UqE)^i{?4LsQz{TH@N?fOOefs-T#^Hzh~P@ z*6dKe_9 zs%M3*>d8m-?+yR&MYgSPxI^AU+jaFXI`p|RJJjiil&x)^HF9bFui5Och3v1T>HzZJ z?*G{Pv~RpgC%^vT=Y2z;|K=WiUHd}z{_Mptfb1rV$(}FRqd*-pD2mf>7(%ve`9v5- zmLR>sI4zl4$qpdLA@7G+?z%3!uJf+*qU%)ePw?9$#LVx$<@{6V(@=_P^SZ0F1vBW; zx7h27%rmCXLG(?Y2JK1ot@e6yp11{AgeBPekuNQ(50?pNUooaj=EuK{=d7Zy##*dL zO}qJjsMCLl>pRr<*S707G)&aykL?%A_0KjsZZpcT9hKOL-Do>y{0SWw*}rG?Y3}OV zAbUXn4~2c&f zFnK>z=>2|SI8Ao{oL_|Od7r;zto9T%uP`*8g#Tuh1i(NS!f)9{M#zdlyD~807u`8_YEjmr}p?lD81;D|v^!hcrFE zFOYZdfpDMAo`-vrLw+49S)+{9M&$%?)Th1w%4>dqtOWnp`%i{`{x<-{7=$4hh7z%yzLh$z;@9lvZ>l3c`DBc3TvS&g+=kB!)jGHgfN2IYT?`%Grk6`jghv<5MIO8Gyo{HOm-{kKrt?0Gzze-(Xa zq5R9fyYQs3Ch3*`=*h5?+>O21kApaj7$V>1R`h<(nq~5+xHyu?{`ASPquKYuKIwaB z`o2Bk)1jLms?~2jU-yo@ppQ<^p&~pHD)|^vhl;`s;S)HC(`aG;rPD>>3_Ww#-i0@P zBJ>*G4uPckUgm0QxvX|H&Eq&b<&A<`No^@Ei|xclWg01$0blh zuU*dmMI7~b{Czw69sjw92k7%EI}HO+j6rC#mtFgO^Zb!P=Q{HXkVOuS$N$a#d%|h7 zpcSd>#bJnavReznFf!#lE!~b0Um~38%ZA4o`Y7b+S#ajvxYN9cJay44Kb8E@CeNVm;XGxb@qvH zu<5^qA8+_C;lSDF!;iZEbJ)M;KZhT#{LjYvO2eDWUkrQKy%>Hld0N=B?!SibZ~Cuc z>pu12A@$)A_xs4@r^4Iqd^xrH z@P04Lhhdj-8UF$Hkpf7e>B<4t3G%m#sf6m}YGR zoAsyW(+n&Pl|xIdKb#hJEd8(H@AaE@j-C{f{+Gf(v`-GZ2Tuv#n>;D(+BV5}>Xfke zV*il++0fAH_a8PD*;}(Xyt!#W*tdK@_;Fu((I3zFEr%1}o(OO0C!M5sf4MlECZo0e zXUKE7fJ?Z7rk92o!x|D&wKocm75u6oUGRQE^S~Pgz2D<6@O$mM#?T6e^RXH~-4l(y z3<~x3_)K0L5*qpr3P*05qu{vMsea)n$N41+J`>(q>09mpA7%F+BV|@6`v3IOILjNZN8?;r1cKF_JA&hwn- zdCvKq&-rPdWBt8R#-ACVzg#>268_z5{JS@_>!X#PT4}7G|K7j=ZlmOHw|if>>$vN! z!J!*H2lXN7C+O`qHw*L0fTho|UC_t!Mex=KG(2i6vzde*%OYeDl1gToN#q*wC6$>GS3vnDP%3K=%E zN83Se=Tl*<Llt8`CIG{ifC)$|L^4gtLLN-8vjVo zI4`U2(5W7gs4IWp_{TNktiM_w>Q0o0we;jXX?j(=5`8058EF>Qf@b|jHD5Bm;)C|7 z^ILkmwtq72As+V*-$t*+4*X~JgXSg5k>%S$K#FK=(WPi4@Z4L|iv8NA|M`g|Fxb zr60$@ug%9NPvH#CA@Pi~I4|yx)gp-&d_q_$~MI^{2v9n_h#du`f)iJvDInm>o zl-LJ7nLZW8JL;Xtzsv?8u}J@;YfSUo49r6P>-xX-^HPi~jG^u5nTS z_gQh?8l!D}OlX}wCZwiYTmE+;taa`>tj9)dLKU{5>#Y9&%liLs>i@s0|G!WFKME+K z&3yg#MPtHIq|tFp|39+mM1%hP#+|M&Ey=j10omfXwcGVGV?wQK?7%MU!9Eerju^;cv4+Bb?6sO|Ca}Q5bIgJdGCLNJhqviMYdh#|KH^Q-{b$^;{RWiH<6V;J2Tq; zKWd!`@$`Sf9v9@+y5Pn`m_vV)EcJi<-aL56^RW<%G4Q1^VJW!`E3p0Fe_ytWT!a2E z>k}YjIkt|B>x!%=H)6fKUi}C273o!o>)%Wxx1kn05N&8i_%3>`k!?B5N2c#Xkqs#z zU&q$4Eqxuve+lpZMtL|yZjtV-eA_oi_%0on^8Xi=v-BiVNMqoc^3X{p#GfOJIF6Hu zYxvg*KShsg5;nP?GxT#vjpqNNWrFsHDST&Paopi0@(Qw>jin~9A)~*chyNeP{oQap zfVg*1k*zI_1JuUQFkAckR@Y!3lY7P4Ep9v>|99Je?&3ZkV$c@vrCS=+f3}caob7x@u%b4=Kezt%VsR|RG9(_D z3q#gj_Y6kcYlXfBYtb}Q`J`NUyl!yo8}3hDU+1^=*obOl2TJWvPy7C6YKs+Cg<9-D z;Wy8OUF06re9^uJ&**31Kc@kO4~KIb05AtZ1FNu*HtWf)Cz(SL$8i$X z4c5*l&)^&iZ581H8S4g@$SXLm4D2Vbq5tnfxIxD84+G?Yw&S~G?Z@K&fEA%MA7h;7 zGr{-;;px*u!#46Z9%9f|-_Qg8|FNOrDSA$yaZlNY!Vr1|ijE78^JK4ffZ@XWwONcL z`~F}pdvYwsV$ruCYxPC@m!7WQ^j-Ux)#qRRui6})n=uBU=d5-Cl*-Y6jped-fz{dx`2P7f ztpBG!zwxF%etr2#H1nlf(25k+h-WRDLaB5Ej>cI1pH*C+pqkQ}iLIK-|0ZN&iTl>m#+-8fbnSiBTAf@#xo&HIeLVR_>Am?B7&!8cP22 zRr&9x{@ZxYm$Y$w);OR~Yq$Hxhr%pjl_-3l%_kp^|DR7^h{YKAgR%d{|1YJdmEp_C z;qvYZauu>el`&6SuYjKG(>DIB`T%_$if844>+1iAZRO{reU9+;*oaN2!Zy^RfxT*E zubSAzB!9HIkL|h123_T!p5WiJeeEx6U+1@`+1d^=+TCb}J4YM;&-Qk+w>@NTHoL3s zJwJlqJ&YYjVwinqU1t~eU?1uiu>bRn3!oD@7(A&0o9YF>VP;(y2Iv-4g5d$sa^vv)(rF&dp@dWtrD@eJTL?jpNL9btm? z2k1G^p@*G+NFTJ#y&o2Lw`;UV8?NpkETK*?gsedQ25N>%7;4-s>IjnV#9}-Mp&Z z0*}`*YNe;pj23L^wpIaO_)YC`u`j3Q@BUC0+Y1Vxl1JQ6!)o?-1N*yK{vw-SlfN_C z_w)Mc$jQ!`ifNdES*XMu%thCtcl#gpCFIaMOg}>TBaussVD&xwOZBHxX$YNd@N3QYPT2%kyL0RGFe<)i= zUyqF#V23u5vH!n{?6-c!HZtb@t=NKS|NFGdm&$jUun4(!4{)NJv*e*2*80R0dW zID!Fn!vXzDiRs!j9jA~+C+hEL=bPYp(3599TXmWu{WxNssnv1Iu}5{CJ;F}n6wYAa zXAjEGkr!|Y{oi;{c7^Qw>>tYd$!oZQ0o+FV>N{ocZU49{EGsN?=bf_q^xVpaWjz<) zDSJq-{x|RB)rVyTVfhUY%X;5XXx3B56e1{sC!s8QaqzD7UNMbtWF#g>B({KO}vxoQ<0kF-h{PGcds+wDVruN zef^!X8RRTv_r6nBNzOq=cu)H~Wx4Km%H}$rkJ7#{%7EV8@08`yFk1ShwMkl&(%wwA zEOHH`u+V=NV=0zl1y*4V)}rg8@&`RPl|OyrMIHqd(e{G$yU(L#@06v{A#1 zdK~}vplqFZ3ZK`{N^U~^*#~8H!mH@X2M@}c?mQ^lMz2LmJ-n5jZ#n;k)?Om z4GPD_aT2F+2Ip`A1HaXNNA~GIyF&JVRo#NzqTak!A8s5IbHj1^o-u@E%~$mM{Jp+( zdUm-w4(`%(^OPHWmHYIEC@ND5$a{{xi_|x^yT)gH=hph{|95MKI39uu497@}!dQ&o z267@MW8mNP(1M6jjTjHeGWMnN%_6$miN{!-(OUB$#{?G`&aqeef9rc zc^LS1x&2m*=a2r=Y~v68FOFMTNG`@wEJNSto%?UrJpFn(n{OWyVWoN8zftd|$GP8)~rw zyU?|P|Gn8Y$lPi5i}UOjdsZNej@#JYc5xp)$j>56>))}P(Z+TnG5RU}Iln90Bd&co zfclq>|DXO;I7Ck_V&mqir_hg}H2!ce+x8M0C;XpYzE^gr?q1oCcRwgQxZqydkCrH_ zmp>@`;ow_k`;`ShsQd3_`&Qm6`~GWp%JxpbQ}(^S>#p@b%D%hgcV)YmUoUHKA0682 zMu+rU&xWHfnOjr#@z8PS*^rFaLK<&2%43HImByO=RKG)F@Lz#Vu>v&%!nk+V;Sdi_$3Wgjue`GX;O;v=Ey^aypEVWC-HQA(dt>xN;*Aq0|C6Gss>eRs-ghy9PT>Y4woZ2QjtR`w5X{ExC7eYeWu`g14!{*U(_lA{$!b8i4v3PV0ZVtWSY#d7zKsj((NV)(vz{Kd#{h25=j9aUWe9^wVtC|Mawe z8}#~5zD)lUinI08ysn=HN0COyBK2i7%Yw8#j|J{N(84rm(|*cVQ3qA@;G9%BQ?FaU6G@*8hK=pN|}0zL(FR->d(B zH$NY3^Z4I5`U?LW9rO&c&g;BW5fZOeghS$qYvUaulc?ACUMD;ik9TStK&k%mRdr{2 zD_YQuv~$wYmnUTv_|9Bm;ZMyVpD>(YJ^rM6A zk9$L8(RrTzZyXwKh-Uz|aTlfW|8+ydeR?vs4HzHrkUpr^`ya<{uw75a{}1&(wDm@N z_>|v=paOC4_?mCZE59obBOQ;zSd7O+OvY4{=4H6Y(mIqAycg;3JZPUpGUK=>@4b4? zU3K0wy)eal9pOED$N32rVVXE*U=}Jd2XiqW3sG9v!8>R;;W=OSTn~D#o^NxV=e$?= zQ1y=yY$K8j*v40+4XyOl%lbd8OT1V-OR)^qUFBf~xe9C0^+5Z_OP)V+=*9cTUu*y1 zGaNmw{o{o8kIU)?WY%#f+4nE}{}1cS|2_Zz8SDIua~;-WBQ~K5g^#O8k*dI9;BWCu zdBgFWtF?cz=cV?q*R1Py&{z%Odyv_y?(vp(8PqiR|8KmH<$M9@ihH-kF_a1A<#6Y2 z!#*59{VV$4m5+z$No8ac67(ZTZB;h!S4QtvR+C9#u@4~?C9?ERGKbiQ(8H$3b$N@9 zkE7tY=(snd4nQ_s<$vGQzCb1)u>a(f_J!s8Tm9!0&fpv_;1aH&A5Yf*dBBgo!~Wl7 z|Jm=t0`{M5o5220VgJeWZ1$hbIL?x#^?zP_|N1}I#91$_4maq@>+FBuQ(=H!(59Ml z-0HZ6Y~~-|79Qt4mgW~2m!I|i?+Uw*jJBDc1-^?3#@{&}^lfp_i&K0jFZfQ-)}np* zg!bWM+5*r)&zu&|E%_5&W%4(Aki#GE-}aO^hoAz(k&sVElA|yd4GTP9?I%sro}BLa z%Kt4_^=}~+^Z)t5VZ8GuVlt+pelY)0ewap2u9P=?zcc8wka|_#5cYWepGskKFcCwU>^?P5E2;phWbkoN1T^L3TbpAha!%n>s9XuJx_Z-cryQwUL5TGX5JnDN5-`RJKpeq zk)?O8)c*eqzW>kE{kyaSeDed3+OwuVs(+sp|0$e7;Wwkh1@aOGzT{c{*7#6*?ECB| zui-}g4RH)~-23D<{qg*Tn$LMI`rA|9e{2Kd-i79I(!X8*_ip{fNa~Mn)-T<1UjO%5 z{lq8CZx_d1+{Z%FaKHjiWpDb(0u>61}(TzEzQw?6G&a;mVz@-bl=IRo{>^rw=Q zh+`O<#FHFs9Kp}jceFpXj+QRR&1B06{om5uD9ueszAgNsaJ14>$Hu&G{hvAFtg*I5 z?bofXLZ6R6yZ+D1|A+pcd-A;dTPU8zSc+xHN>@huR?u^A`L?#&2Y|i?MaKolagJZ_ zZtc3l)*-RMJ`QBvvtz?G2Z64TI|3s>_PQM zt?f@9z#$}X1WBaOi5v=Fd*qXp=D)mQu1hF?wEx}LNC;OVmgp*|dzmEy0 z$TK*H3%GA7)_>b-@ZsFyn~PVyZH%THGio~i$Dru>hlo$~*7`Jer4AzKlBSeM2d)OQLzVaI_+YqW>JnNu0tNoWliNLf3Zb-{~I7xL08>J?;%uP`@d?|69_J zqe!2S{$tXHEIN_yR*s9OAJ=dL*$HqV_(tJ?>d6crDgpBQ{~c zGpi!EA#vIIm}Jdw0?`_G%F`S^V0EqvyF$2U-8Zewj*eBS!M(er?xdxw9| z7tSx?r!C^AE#QCiY1{F{|9qBThwM;(-bMY_e6RuW-Ns$qNBvp--+p^Yk8J`?81x-w zK2n$68(I+8>Kh^~?Y=6=;mB?^o}V0r%>H-R|6gq^zvJf^x@){3 z{b*hy?w4HyajpG{{xcaGh3|98KC z|AV>WtUn?B(l(!-)GiR$|6fQi@GqY9|HpfssSj9q9P^W5e-_i@9J6I)%{ROsb3lqS z)fe~~`3c@Pa`d<+=^)Q`h4WTn4XVqv1-)g>5BlTv@7B?apO_ielTF*@$D6*{d%j&f zUjMHDi{_gAobSq3ZFJryRAC!xu>-rX2m3Ja3H8-a7|%fO`&-XGBhSBTj!2n2|FpQt zIL0USf@{#@x5?U~+m?p4qDfoDw?92d#9 zDe8aI`FGAqlO2xVyZ+A>;c?7g-2ZgIUIR64<}dtOyQjDZv<;N@|I;?m?>NqfxJE{Q z=SDo%UJ&9(T?V;Y}~If9yju z*|LDGAXDr{h2Mr_Bob$>g+Ru&>FU|cvGnnH@BHA2jwfR(V*mRzat3NXr<_!7&Z1Xh z4(6hvk^M!JzWgNOoc}KaQu}@Ta{buOOHa^mJzKvunMLOaVbZczJnOI?8xh+u;{Jb|=(+vu?_TQ<(zl_= zt`_dGzqi<5vQ}7PgE0)`F4VKBd&qq#?f-YybG@t$-0>l_<~`Rs&zWq=#P$#C|KCwJ z_uVAPX5USV@1)gtks`Mkqp|h0@8g8}^_P5C-_TAep4w*LioU5X<2kx7O22U)ajZx0 zZeu&7yRcXK-;(|XzP}~DzvaF^^3nWL?L@cCMMrvr?_PW9Efi99JR?m>BqtTyhY{+2bt>E{slpls64l{{~pkK;>db)3>y)l9aW zC|#R<451NmEz(rp7-h64P>DIHe%@UApO~jkpO1lW&Ik+1#Zmt#e}G(unychJE`#9ln3~(_!ys z&tv86aA2@?9{hG3CvgftR#u%M&*1_t;R+JI!+!D_ZlGcJi`r+rgPSjgWS?h_mh0ZZ z*%!4>ncu(2{C*ro8Xa%E=oy;d|C;&z-doo=>jR)?ru6~PJIVS0BdiZF%=!RBUkqE+ zRktF}pB-?$g1-8`w%Oq>z5lb;E%>y)Yx+YB{Jt^_+Qp8b=7;5_{g%?wR{g>9Fhp1d zh9mp*Y~#=56?*QDz3nlIJ{HBx_O?gOtKzgHqdpF^hSSr?F;kGPjm5xvLM4bXE>T>%UIwipdl ztUqMDL23P=7UfdL_=CK5A~JPR`-ZaMasS3r=P$zw)LJX1=EvosOWD_rp6SX1;$~;h(5sCvDEa?vc77)N%eKb7!xQ4Y z?fX3EocjId1dx|dcU`*j(zR3j!ROS6zsk0-)g}LbuXJu^f8BSZ-<$leH2$9+?Q&`U zT?^k_JpH(a8yLWC+{Jx#vE|(>mGQ`-7kNBh=d10a^y8>>#Bn&TNEBB~^BdB?S^6fbZ>9CM|F@mR%nl~ zeK>%`6!V$MI(6!L`8h#9f+U*c)g=CSeT4pBSfl*I*6*`cPn@T><#p*nEKgJZSMzi2 z0$&&!I_WtKxaWbOZ;E~#CvggAa1Ix630E-iv$+1_(9pL<|A6EE4_il;yb*uHZKUt$ zZzAvGAqMT1es#WB|9Fa?TjblM5207!i9h&~Z&mmc|9AGg{_j@)?@s>jcK&aMe<9A{ z{x=e%@ZR>nv5vbA^8Z`-zjge7{(djtKfj#+Kac59R+p{h=_Go*c&irI+Tv7=x91!@a%k9?ST@!lyZB24*3hsR)&1 z#yw=+!yNZM$MIbB80()yufF{!_vaoPUgd+LNt{WUB@Jcu?TjuA~gfOFF5L=MI95$zhjU7SSMX8He^Jcb-fcJ^&~O#UmPZ9Dt-vi!e5 zUL`x0%bR4@aVHtq|2ZX|GdPC}cszdY5xiTR3H0~-ra1|K{kqgjeGsH&13&?bU9ncPIW9`+itOc=tSam;|#=;z!4-- z|Elo^D~R4>60#}#-o{%EsrV1xI6!ux;Q z`@iV@Yu{=14YZ)uHx&DdhWpP*jKWxaC;GYML`+84P4!=Syhk~b+p7Lc&(jNe<4lwd zZSrP2jyf-`{@I~SiF*TP(W$J7Yfn!V&os=yEY!c@{rfH|>B)ogFy_$b;>q|U-+k=A zit8xN7rtdRyZf{?zv+b^8UsfzL-ogvfg@|bVm#7!o(rq!Yp@pUupS!``{qjH56q*3y5;&~@7Qf%C=>n8TkVd(FwuKQMmay72>OtF!)myY=Uh z-f8{*z2@)VF@N7Von(Xl?nX3C(SNTmy?KKEdwMIG66ZtrIOzNG1BRdi{qo6hawO7M z$A(elSd7O+WN*3#Cew4%rEQ#X2J~qtzAkP0)$^{|>l$yqs{P^c*s$dl-`^th?;FR4 z8P1!9I(pM;_vJsOd0fBq?4>)dYtX#XeR>WJFSvg+z2rQU=H+`Xt^SijrFiCGF6Lt) z7Go)vp=*-VdOcbH-}_FZLz$mJ_F3akkoJyOh-VenU@fx3 zGFV5?4fWk)J$)mJ^aAqe#U^11@3)HFhWfZy1i1rs^w{^D#4g8s&`M9?alc>5zaHlM z$3DLg{8FD7c?i|t)h715>wQ9-#ILn`3aj}rd;2eZ6?zKkmFf^=Cvqqvt6m@17(5=s z)Hh;Z;7R(a_DLg~1c++`w!9$! zkT--4M7}&W+$QhhJ|1GwUimw=AH?*H(5C3P?>EndA!Prz$A${BwCCz~wE2Hdd&2YO zVWhB882Fhr{(nCv^acE#{Q;i0PDk6AFxGG5F%gq771K~%T^{28f-~qD?TgtFWnmV* z5_9lm{Ma37f8ZIA#hcn(*_y^Y8I z_l5P2HzMx+ld98CK(=J~TdzGT_v0FzoBUpdxCZq$vKBR;<@bLngdOx<*n@pIfJ1n) z|4+B;pt(`{(drqbkZ{fsB#}ZIoyehxF3-FhrFr&b@5M)TtAhGfk#F{7f26nAe{?L8 z|Byu|V*kT&@tnjdoWbM%hjaAYW;T3-HVyhE6!)?_$a_w`xFRg}KlGE=5c?l)kOL_7 zKZrlBD_q^|f5tC9-Z!@SS?NT>W%d7?Yy^A!C-ZA(4+?k1a~}^;`0_Jh&<~WQ7=j9P zsq1&6=c4-mS#|Pz>gFiySO4G3{;T7+tLq<)b$$7;gUmRNV{|&5w{?j&fO&pf&6d++ zzr%3xjzpGT^8f$sQQJb#TjJ&q#Cm_p4{#iP_EF9qi}9F7577W5}4tK#zMv#Wf-t4$8-SeYf;d`_}8;FE+VG6}F)kJFp9dwr9c~ zvSA$m7fndce&kcOOyOhlE#qAKRnFOm12}{Pjv$E?y7cpR^R;{U+_~5I*BkiP`qm3% zk+0pxzimIv|3w-d$TaeQck_Rdrl-Yo+xOK;=8$c14vO)3HQ(BO9jBi}@io4+u>1zT zHe$X{+~gO<`X9ai`iH_P=bb?vy-9mP>_b22_ySt#aUX-n^ZXlDO22rTHam}O7Jthf z>3`rnvc|ZxTJQ0a|6V~quHgm-@V@mQ+11$Rls_S_p_Ko3O8;KzKPa8j`zQPFy2fqs z+{Jx7M0SJpUp3}opY$)1{&^n?Ptk{afaf&=LflZa)o5>c(tz=3*o9sVRF%2^?3zaB* zT^l>uwNw7*i}xT`CvWEE(GBd&X7+__yDtCZsJx!OCI4TP$C2%p|1N(;K;ovk|35De+dI~UrOsQ1dSP`~K~KIW zZKz>iYh(M7GN=_TtEE-=D(CbEV^F^MRcqN)gf))WVjb4waO0rxlLzLB92*pVI#fMG z8}tC*zF(QN(eIm3h3fM1P{1~N-1DoJ+<{%_S6|sf_I*Jehdh8oNZ`Hw3r8F$k;1Os zqr$+C&Hwm~vgu#s6=D7Vu8oN-qUNL8b-r63_Fo+teux9ttu;7vNI3YwI*hl5ghTS< zKgpN7Cyfo?Jv}PynJ_kdPhESjwE@3BX?WQ8>_~gD4G-Vx`*7Gf;ltq{r+hf7|#{LxTz^`oKo)R_ zqqF7KX)6!kx4y^Ts~-$Mz`o1}!xg{v<9n|te{gtr2-h4Z%Dx$Xx?lfY_cz1Oc7HP* z+5e4D-?uq5Zha#(9R5Z~zV-FcRQ7e>?AJo`D_;+(?OzM62fr5D`@S05%D$@q@GIfy z=&$k(z7jfcL!1M+jk~yyhZyuj_6tK$fri<}A7}??LQ=ax^JVpl6UHAPHCns7bKdL! zk90f=V^Po_-^GS@qi2$~2yF(vcT4|7=N9p8wBJY@VR~Z;|#tq_EU~mSF`}q53n{a3SBPUHHX%K8aHt|P>f~AV zl5O-_>_E>w@uQa?o=2QRTU^br@LT;>_Wpq}%k;!*HuQ_%3^l+0W~lw~H??6&%Svf- ztvI$fuHg~)`aa;?L-=#+|8Cu`O%Ms^zIl@mLMD+y8t*IrwX^-%^?%j*-dq3c6h{t4 z97o?L)knT;PQWMBM}DE+_OZ?3r0@ZKR|6l_Cq_SmbGU%?a?j#(-wc=NSJ00*t|_hs zevO`!{F4f+7$p5z7OozsiN!OdZh@Y}eH`=~#^In-q~hllj!z0JO%&0)}f`39*G z-wLg0aZWSdyZ?B){aay(b26)|^gmaH3VO|Fm4)T(!}Y3=L+`yR^}{NCz*V7mx+)BJ z&Pa@Rtx@FIsAD1~qv5r0g~sLI3QbGC6_VSlLi5(D(1KQ^(DqVQXva~c(XpXQ9rDrj zJISu)RiS%IRp^=E`mB&0R#vJ-w zq_#@`Zt2f_OJC5p!hB&1u^3CS3@fk-YfxHmNxM|6r>u3n4%PaK3T@_c(i>Wg|M5PW z@MQeiv+C(jv(qTG8MzUZWGCt_%4_%hhMeOfS~s)7(e{!p8~p#D%foTuCsFvt;BbmOgLAlm_ddT% zj<2BFc;sv34a7M-17ywT%1i4E+@{~feLTdVANg)E1QqBvzhF4o_gU*^lA~hS$E(8F z7^b`!Pfo;SOhxrq2Zw3o49r3$20mI9=8$udzV#PjKDiJ}u?*Sko?oA}L+H8J`G&8m zL)u5M=9lG<#$&!B|0xGzUvb}OtHLVh^?&+X+8@iqn-}GCdjf2!Q!k>giNF2aw?p>* zi*Y>UdJI&5D{Lg2lsQS?&L(;lO6$8ERKHXIjO)8>6P8$FO=ofk>KAz5=lo>;iG17?0<%e=q63{~-IQ@wwO@|I7aQ zJ+6QA=5l$sj34&0=S8o7S{teVokAVGX|VeqYW#rXb7-Zf?y&PCq{Hu}`Eg5}hvZw* z*DgFSeT~k;asRo1OSpo5T*D0vplcrgA8|}=4o~KPsAI>nw#_qo@_crBKAvR;S$gMo z&&=<)#d8<;@er|YkeR6+|A6$rD*ew|=Z8K7#pUcj@~=q$>&6HOONehcITH167!yE_ zMcroUa&B^pc^r->qE%ceG*6e#(b9kZ-Tr^yYZA@V<;&UX$Yg4Y`uS--`n%=VWdEOv zI6tWHbz=kQPsZE7rGCC$nQ-utk6zfSey)DswnY7Zx%xku<_C1#Q~yVn-r1*K{xN>> zhm8vrS0(0PF6Lvve7=QbpZu_x?6;R+;XjNKpfAG;#IZa*{EC8#^eV?i*D4^7Uh$`& zHpWKyTCBr*WG}l1Y@|1M$BihhZ`JL&&`a$<=EbH)=u>k}6}F)kJFp9Tun%3&^4q6a ze_+BRKfd?kBfq_Hnje3H-~QU*(7u5mkMzs@_T~Kc+14MJ&X1ooIP{ql+4lwZ{v+&t zulXCE^_IpE4v72BnffKI8E}YR`)%thSSR2JJ&A!|nv-hoU}BQ@o&Ae&oPH9i&FVvFLGv-~|9tC4K5El;ej*<=t|?dQ|L^AiX7o?{&ncY2Ib6Uc zTtPp&^7_AxukYde=g3~Zf1WHjE|P8M`M;O>zhue(-^;gkoh;eux@mssHSyfQ0B$4B z4andwJ*OYMXV{0refl&MU*Z4riA(-J|Mj7;1V4DtkCpSNrw<`3P)BcCB3;Y*Q9m0L zh6`&QAzh=Tlm8so`Do^U79?qj@oCSo$CA}cK8zNgW1>Yma6 zpFy96qT>Sc;_gMIu!QF@hn$Og`h0RB>W01R{~yz4<#;Jt>2dB~^I7%*rSWgG`N;Bn zQXX#JFaM*Jo|1=``OgZh!Wyi_I;_V=bhYsRSF3-#rv5?pZdU&w3yzCq+uQO#jyB4R z+6OxF^4{S`{(a|OYOw>=$Xsx|J&?8;+)8|csh|o5!rYS-~4zCbMHMkNk0{Te^%U% z^XNtFS4tdK|5@oBA&wn5;&lK;EYH?o)g@Agfo z50u9Lkg3yg{-1v4FIx*M#`&c|;R@N0Yq)`~4f6kHd5p}-=e-x@Efna*W9)yP|1bWd z+xZ3ZVaF1AbGiIab~=vZe+I-;{XuJTl6UcV{Lg**;aPQ^6L zK;gH~g;``J=3+kjwmcOUlKo%u+|@bbbAJ=*&B|Hfnb(x>tBuX4*L+NW=od$ao_p*A z8&c{Yx~L3qXCE-|lMt3UZv|Fi4c1~E;(jO9Uwtn0{Zjvz<2e7X-#FHd^uAB3^OFg2 zR*~C{%i2cPqW-#f-)C$BJ?WmBHrVHdz6YtVMvC-GBethXXi-1m3(S&H7KbJh1+CqyB&N%+r65-dFVBzpVd$f&P24?Uwof_ssu4 zX0AUvy7m7f+oJ!!&iH^~`uA}ZX>F7p$Y`_7F4w=mXl&?u-S~hvj1PE8{OH}wz8qv< zNOMxc5%+V>w~-`MNTU-q|G|d*O8=%d&qm`4n$FsvK5u{e7JC-}apQJZI-Ncn6JJ&2xeEd$n(v`|_4?%I(IF;h*Kh*^xEDHc7$JRZW|O#+;%z2d#M$basczRCY>b2R z=3^liqvU@JUrLXA>h!Rg%jhdme1-j8^sfKC+WZJ%iI>!2$hC<6_d2qgk5C$a@7cz6 zcjEqH8-=&JpI8SejWZ;ltiRx#I8MKLgXdr8-L!Z|9Nygh|AuYobeuy?_g{zF>m%*I_19th{=W|YaQLsn4ix=%9RH}?-Sx_t z@SXj`!_Hx2!uMYPo3Q8V--MHX8~E+V!w+tKAnfz*zAtRA_x__v?oE15`TY#eq3;u; z!jCU&`}&x*-^N+PQN8xybYW9Qg#-4*`q90yVgJmr;fL*G!@jb=GKXz=*sIU%`-}cM z?9p%by%$D?-R&d8cPFh4yB4ht{l*Vemm8=5obf0_y|3xs*KB(Mxz;6ILE&Te#UpQE z;PX#~nlEbi_>z6`9N)%W+(-4_SA>V;fcCC|>UYQgwXyrZG5+t%+Jp|v{}_S_497?m z0$<^?`hMwSF&+~!8B;M0GmudK`6&)7dw(Jit#_?{%<@|$=HSi!(y%+28^(U>ziT{b z4Y{3q)J9MgA+-fq78K|cIm z{ri05^la<@yXJf>#A0OI#o_m*^xSp+|9<`O-{u$i?u-2M0`eEdb<4g0&Plv6lpRo4 z+G1WZpM8a}Rj515|8;J%#XA0u*P>NiDKuxK!+jhbJYB!kr$XCnpOlX2VV!f&9v)q}N+}^o%&pq3*oz z21#KT9A83fW~g#)XlVAImc0ER)&Cn`=6@_#PhXKL!_Kgd7Xf0z%_j-yDU0~ut|iTmPsh(SM|ia?=?n(oQfHkg*dh$)2?1b&q-SkO6%|aOq(FRfXDk6$NeDtp11zz z2hH)<&tDo3aaQ?(IsQKv^RW<%u@uWN@NHujp4UD%LVd~c_Fqg6tH?E|pK1Jq{JoZ* zJmJ}59eq7g^j5Se-=nYBFqC~{lS=KxOV}Uws|Bsy>=dT?&qi!Q6}F)kJFp8~3)tr+ z?DHb_`BnCrPoH1SKEK93lWlLa|2TS?pL?GFdz$^i&D{T*|E$h@t7;`(uZLf2sJ`$N?M$=qmlJ~HpPKo->jJH?qp5yx>7 zr=sqgIzu*0@qDLyzKH$s&1YQ~twXRJv~LS>e(FYHn^1*X6h1#Y>>v||-P3136XKq! zyBzO9Y>TM=s&NVQIJP2=!*97G->H+N$hgN)jsMhkm?Q3-LrCCAJRbvp>s^t3|Kd9# z`=60#$Q+6|jyV7CBzX#Fa1L>Ap}5BD1$t?{mr2I2(XZgi`UB7UmM6saU-QeSzv$jx zR8M^|G&^qD{$glFs@r^H|LMmy+`xP1{|z|qa!xmT=2i0lEBXJGA&%20f%GPJdr zpPn&4z1{ruo#v*^Q7pr3gLyDP&}^SWHkVPlDu1eoc=_7Lw%%7*{FPlGq-u!vbaI84TVS3*3qoL7lCn2Y++((j!4^yEzcec{EhkiHnH!TyVuVbbrsIJP|2 zIhXn^jxkt9u0U!2--nGmjr0G4aj3%9Vjb3FBR1hr=Krblx1h962Rl*aoNcJZ4#fKZ zE^-g{p=&hz%Z~o1^Z(w`{)eNJ8c^f0TzBZS}RE zx8~?G{D5DotH=ME|96SJf__}X4OBm`&OzSBUBvl+_sNHd>wgWZQ?_CVD)4yyui^B^ z>wk@;k3vcr&>G7D#P;{G!p37FCSxk5VFm_1;oBhlKJ#LzB>O*I7Pd@M7L8C|dhc_E z#rc163`HC}FyHY)Jf8oznErVF-%|QA{QsT*cTO7T_+P31Z7k0U`YNo!TKsS4|Lta@ zP{jYu{J(YLS&xm_g!)$>&Ht;SKc4@$jb4jCp8vN)SX$e`E^-gDx1I|7$OFjm>w5ea z_qRFZIDrD6zlh!w>M3N`tJ*Kn^Op8|^s=RS#4&cI`HREU4@RpWu)%5Pb+EY^Wa*tN z+0S|G`;zz6pJKg&4Q_p1*^O16-+S%H5%-WpD&EW2_zru;Lobd0`sWM&;Q%{ zn*5EietUDUcF(_mCQPJH#z0(KU;Uu(Idcph_y6IkFpZpnnqTw3t*!Ux#{bQ7ZYAbm zF6Lt)7Go*iH~#PK_q6ZRi^uq18RcJI`RDnh@yGK&mWd;_9jqW%q43-@VGTK8eZ7I7 zyzBq}t?>b0Qy=`HxiNlQhxOQqIDTsrS%qz=#pCgRJLr$c|Lvmh!=H}-8ydz7--81< zgwp!&{Imr9@%rzj_CNhUsS$irwD9YivBetuTdlMA=3#5$*L^4)@t@3L{-wG0NqP!t z^t^0*fbqz^WZpUf1+r*dbpyY&k$>8>lK;Dz|GQe;nTpVwxBfoa>AyJ?aU3Uc3TJQ* zUB)?gZ!rGAxPu(oYurJ8nD|jd+vy7JixvD2_eXX(&Nwgoy10?JZa+owT*4Lfqn?jk zchg#H^vCND+@KF2rEQ{>pZ<9Lf!o5~o4@ZmzK@59^Y7!ln?Xmsm)~f+V`rYC4?zXe z@=wES(*KsWS#7e(-P&e-GcCTI)_dCjQ2i5a^v)fLQ5cKyn25>f`g84nFKPe7e>(nP z*k6RH;+cjSn1$>-_7|1(+;;XCbLewXq!*Cit9=mLKl3*sjxU?-rX2Lr#gPa;|BfA~9X z`t(CFe2cb6vd=p>LiT^r^NliSSBc?Y85+`L!|U1~(1c{8`|4(!(MnIDy4g4#=jBku zah${{oI%%X@9_`S2Ya99AI|0mTRD8OQx>RcDEP7Z)7IHvCKEm8jb*EY?2;er4Q@=`){qc9e6ZQqRW@$_6f+j`iVH1x?R?H|0GjjdyA8`)3SY8=PL z;mQ1addW{Er~1z{%)l&EVh-kFKDw@If6%7iL*}&a_mX+X1+u7ZqO|`1boQHFjC1We zq^mUlUfMd#;{1E-|Eo`J&8S;pp*ZUgTmN79VtP`2BCh{m{S#y19j9X7fjUKt_?z`l zEE5*@6)LU&|55KXV?QKetB}b{`#d&%iTbzWwJ6N=+|m28_dqtduSRu-(mFkh*!0)g zcC>cO|A^(sI{#abjo5@LY(p)&8hw9=V@z}CeM7t`(2FR||2;4NBi-ox6KC8TK1=UB ztp0IU`@VQ~VGs79{ucXtZ*Vw3k89pE$;XH23A}IoSefEivJK8zeN6$SiZvE&7=!Z~rTyUHxd-eas{eKhfW5Sjt-pgw3f3J)Q)z*I~{!0HF zz3FA?TJY}t`yWcXL3%!@>~gJQye_ii#4+8P`SjfD z(*LTl2=p_EV+x<_f3VX248juR1@aQ=SC0u-$bQuMZ)sfwab0tK1Eu{B&Z=j)ZruN% z?+bqu2IBdD|Cix5c^CKb5QCcJD-`U3GlZoZ^5}oT3!d$icLxm|}`3qAOZc%wmdJOc6yCQRG<0lIws2 z3Mf<|3Mk?VyhOanJhkFj(*-z9Wx?{)NXvU%?MUa_gg^_rjy0r{D*nTbr(HmT%|0_NetA zJI-FKJ$J1!nWu!fF8agE{+)J??~m(0R`A^Le*7B!*U`A+H^PaNzY&g~d~Rsib6dE_ zy?mhXm-Roz`GbYw-c7#}K7W4N!}_~Y(pgyWwt49CMY;r3?nxzfUea+dm@6 z#kcS;BL4_~j6cQXA?>p&`J4V{$dxLykSS5uS1WTaSkqbjpW`p^mq@#wl=#1*w-@>U z4|z9E;Qi=OzbC|hfZlV`7}mJC3vvAaecJ!4)`me`Bc`{hFdQB9zd&~MF@5jRzwbki zNmc3J-(l??`p58bOsGRAF@+4W$f-*^)t6o4>c1NGe}g);Ro$u{@6$)$U&3y8Y&$-U z9`s`vqnN~#<`)!7zf}6pokb1<#v}$YRBNmOBlIWEPiS|poOJ#ce}}(EM;-gSm`zRZ zNxC-t6aAkN{|V419DO8v@vq|k4U70+xLQA``~F>c3IB=w|A&n${TKZwSlc6Sq4N~~ zpBC-^IqlP!FPF9d5!aZA>rcn_|1a6^di*ke1<%0E_%)188~>j*{*TFX#{XB1|2M1S zT8)*E1ADdqF;u1g#|S;H4VSjx=!)YC*M-Rq>%y4#F&^IwJ%=e|Fm%du)7BYCd5+#s zTvKUO`)Xv_b40f`R!^n(xYs;N^v!#3XX5yZ_c&{AqicLFegnUW-@F8mILXUroe ze;@JxcYi>pmwlfX*%9yP{q{g3gp>yPC9AJu_Z{U1~2-PrHXkjPxF`{Mc|`T6&~ z{#{3Z;`$@M@c+Yu+W+WBTz~I>NbAq>7x+v374F9SF}6efj|s%S_Me;oH>3TJA;fzf z+2r{njZrKZ+dm~!PooF@ z7#?>HGS#L1FOQGXCy_%eBPKEB{xit?r?+VT)tEP7``_a4@b~B#EDDK(#&PLA&6mgi zv!Bvd6OR8*>qD}E{f{Tj|Ksof&-O`?X?FO((i`;|o_Lxz7<+n>y+5UF&alI??C~7g zS;hX0K%7+p8K1HUH zpZ}-+{bT9;DgHbD3~Ap{YEZpFpKMjX;LquQfuEiK*YKkizGv2;v+b`C=bqe6z8{Gd z`EQP04cC731Nhnbe`DvBUzj+g{K8bN^6Q}M*RIQv1B3h*$N4Y1mtl-tbp1%XpHcVF zY0ST?hM!;^KS46i=}*q^pF}@CKs!E-qpQ~Y+u(ha@V^x9NA`E+ z9dZ(J{y~mB@grr3zJR}_{~i7w|A1ll_)p|N<6rS_h-mzxNA{~v2R5h|FjS(R*rZ;tZwhJWA3dvo{i1pW6VvMd z(?9KZ7oYEkA9PH@Ja^~$P5c&q8|kDpa2I{@f@ASJ^xs2lGh~rDCoS>6FRop^^#|l1 zqT~EE;g85aMuOhmb$MR@p9=qX^tHHdYKkK+?)$EUG0urBnFas0NQT>O)DVVE4nBy#9%`Dy)M zzV zV}u@kTd&kV|0az^44-0WlX2efzmrQyi~A4qKQVb;IZ&_ekiVypTTq9H`!Cz#f2I>> zgtZad>G}1XJ*%U&@}ROL)}_gWZxKE8SSI|$ao6-Y&kOu_h&=Ih@auRk7K1r3KKVPo?_vAgW!vu{-NY}M z{C%YC)2S}(LU)yGLw;Sd8rNK{{y*%V-m)69pZZDo1N;9W{s@1JKgEB?*hT9fAm7ik zSDxN4Z||1Jk*j0>7pwnE)&J!1S@r*%vK49iC>DQ}{jUuW|8KVV)b&sDAN(`v{}22* z{t|zMxR%h}ESHlTe0Wxut{{i_CbYzrSkNlY5+>iAwyGQ#ZE8U}g z;=h*q&{=mWbk$$7?NaDLFFPxVzVS=m-KFps()k!Zj!&Q+pGFV*F_yX%#xs}nfBZ!M z$4|l(GRPu_fwPywAcp2IY5%a{q?tmx=~5WQ@FqxS6qCpyy}>>Om%`u1xFTue@92LY zx4YgfGKH74Q7?sm5Z7LLDf|=p&*<29Df}z>Z%FKMUwhpbjz7m*PvZJ^x-azFFS+}Y z`K`)-b?`9^yRQFbpOouN&t3}uPH%jJdKVM&=cIf&C4XkftZ@9lJ)p6TgMu#$9;Q{A2ab)&9Q{*H`TOJEQ)0k3;0}y!=n*|1V08))s`{k;d=g_t8GS z8vcMx%&dlv+0}6TAJ>N03upP@#6G~DLf?*WIZ5^j_mk1*FZy1!Z(z^v=YL+ZHoVa> zZ$`%^dHRuczCU$ykvba50(EpLJCf{k%)kQwAPgbSs7en~6xJGrxiPsc{uh5rXd?|dDT=S*NG|6|(^;{Uq--`x6et8{L|?YIMR z4ZsABKj@odi+3aTAHT@9esy-=xHgEkLoa#M_ZPpxV^!LR?9g5IN!4k89J(gFgx>gK z_48|dKekP7bA3Ct9gwBxwrC@@tPd}>-^=kzyc!+GCKCEGUQ6#e=lu409sTv_zu?)3 zORjiUSa@M!c%%5mXRixyCf|y;Tj;^P>Kb4bHL2^WQGb zozmXw9Q1zje)~Lthww12jFUzCp+k7=9Q*TvcT7&M%Kv0WI7{ZLm(BQ!#HQ$v*U#`F7tVuRG5W(wIONpTXxawpISz?)j3F^6M0t zU!O~!%VA)j{Jlk<5I=1D2$`~fnjCem;RZf9pI;NcAgwqD@JnP`T&j6(_%c2JU$2wO zk*651e^8p|U6ZmRLr%%t4d1&q{Go5OeR^&9iv1JwYr|K`ucJfUH_30KC#eiL?cUlh zmjV4`ECar4+xPJ(9>db-*M`T*OIS1Ly^gF6>&e*uzLt!>R`oT$x4IBE3g3Vm5y$qj z>(uS!X5nWcj`20VxG0>kp5Ww)cjWzK65i3ccO&0s>9^SLR@{c;`afa( zi(Gi#`tTAmj*Ue7|E2Vox0ghtKIQ}?WIzPT&#s1#Q{zm>k zdu;B3W&YQF8?r~;g_zHdU zEZ=2l zR(Kc6zGDZJe`l2KzL^x6o(|!XV;ZCX-_v}1XW0jsn)7^+rRVH-rC#pZpe;Nu{UZ4S zN%SJl`CT&$#1{>vhRIKMB>`HgG;$2CA>Suna& z-WQ+Rtt=3}Nje=({MB35hRyVL@>%39xD{~?!aij|Qkl?;9(w#&Pq+Meo9(yb4!j7v z@DjWfFUKqKYP=S&!|QRie^y?6qj1jH%bUr!W62oMJIUi8T@&6-z85F7eHzzUQ!G3^ z91%W-g%8Wu`TyUh)YlpQYvh3Xa&TGwgJI-y>w=u^+f zvGzxu<1svrOIS1PTg0{4i2V8k>h`Pa4^*q))&E)be+~oE8q}9Cq^&Tl%`oB|DWq%l zPv8b=+=!d78B4}-79QCUo<(1Lq$I?8{TBLy{@JCkJtsd_)vo;fWJzIYJ7rCmD(x5B zZ^P}l1L=e6*qZg>Mf6E^?S$vMi~bViiai@-)a`Nper$hysO87urMA5iuSUN8;n}>F z-gDl&S>Rs~{}1pWdwTYA+bnrbTfwt^=boV(}jzCVt-=%>+$e9=XyOFiysRoc0Lx4 zZ+JX3EdMavbNUD21IE_gXxp1{@4|KAL&Y1z2g@D}rwXK(+!#Jm|CI1n+un}*&#wy~ zt1wni*QpyrC;FRjGoJT6 zV?Z~A-j*9edt!5V7;X9@9#@`yy5uIlCpU$J|HXZ(_2%$iX&%8bG~yoIi~DdthR@v^ zhA^1?jWBTTw(x+shmb0Gw!ZjVL#Ohqt4w)P?pny+?aG>zGAE<`Lq2TZ4s@drSIdua zeUids``G{c+5d<2uWw)zlw2Nr%hGcN{2!;4eY48Id1WCvA{^J|%lFSmDUIWwFAZ5T zuH*k1@^e^vneXktS^G2k2k`q_Y|KFzEt19(PsRxyT9Tzu; z&Gvg161I0IjoZ79TZH5P!}`ixce(qJev<4o=G;Z^c5OX_;vJKmbAQwJJF_|5>bTp` z80R0ptsvY^zXLDAE{yHh{ywdK#U!SXIjh`W*2cqtdl{5=-akMYeAWL;{V*!rz1?#y zD-AD^)=SYR+~536-=Y67Y%SHlV~oZ>8eVSOE77s3G`yO8EfOZB3|DRrucN;nZ^WCC zF7f>ER{A74v2$~HJN=!=Ex0!IP{y@Q6>JXg7WZBpLA(Dz93vC6rR@LG&`7@r_o92n zb&Bhqc3tIcf--G|?d;csm!Es|fjl1AH1@q8C*Glb_TzQoA^OAUKsWj@gfu1)eb=+( zXYe_E0bjzGvGlc@!&k`Y!}nEk@tFnT>*P1_ZG0D#HNNLV+8pxB_l0xHmMk)uk}qPO zdsO^mcpR6Iu9IKRushySsQt_SaW8RhP7m4Zp5j`eea-GKrHt}Ttx3uEbC<^nKCAt` zUU(yJz*v|3kE{KERi3T7HH&=z-}Dp4|LixSjg>;0KHBD=L(k#s@B)5M?~=ct?mL&} zO=uju+z)X4h1Z15!q37j$eL?>E4lPVz7e12Ke4hV+$MZG?!b%Cp*%{QTN8HCZw8X)DStms{SWa!s$8nzq%~?558GCZ^hg3PF(RHd^i0{{rO({5#-dTac$iU zIkn@b>;EP7o9^))?w9`);_TOmg!t}*?CQhHO}5nuW08Hr{lfWq_T)YGTY4YA4)Q+S zj|cD&9!3Yc(T5?VvG^A2y^#z0afWAH5B)Rv9KL|G`%K|W^vN#Irfq%rGW{#asqeGm zGMMswI>&$N|4`yOkzbdu-19GUo$m3gj(MVg@z;gFiErck7}J&u57Rr4TU9o!C?g8JD|CzN!w`RG zy#5(-;o0lM=g392T;qqmuOD9*z9jr*9A*1Ozqe!TzxH`;esKxucR0^i>0ie;5!e3c zLGMZJfAqP&e%JY3@Aq}Ke;eP$7w~;NipS8|YW{!hAK0mXV23epc4>b8KVSWRzUu=e z?0XE^HjI%Ow(lJO|7rF;yLW8W{B+@oF7wkdm0|lP*udJ2IkK~!{f}<+pclzg?Ei+# z{{N4=hD%s8s*Qkau@N`mMvS$x|C8+h1pA+D|D^v9>W3JZVgF+Y!x%}i|1<1=a@29d z?CYDPvl-9AEy%AwDE?OZq_*w^Zlm9hT&-)bb`NawDdhc6_HFz>{wI4khCA$+eJcM$ z@!b`k8;(C)Uyk^`GS98tbKLDYqI0MI7j#!$-mh2MN%S>&KWJ3noH!oBOC0-Byd1B@ ztMOVqdH%`I*!QRPfA|&#eG5b6@ILL2O5f&wFm*B3*6$f!d;Q4}DTyIbqE3Uiy8=XySk4dugu=+o#j!f7uBc4nO_mTZ|eD&DT^~;yn zf6Q9LcSss(oKOcenpZS-R{4wkf7WK?X=QK*dHAP zT^K&FCVZ8iBGdRf{hRnUCQfOeHn_f4*GFcB<64sKN&bDde;41!qv#Nqz+?0{cfNb~ z`tUgY68eSvgp*`%#rm*jOuodm*odq4(asQV5WW#NA@Be8tNM88&%!OZ6`ktfu9W_X z%uoFbHt5^m!Tv}8c7FGrYr}2!yB&AnMc9Rx;H4PT&ow^#|L}j)H!!%B{V%QIa`r!& zDrWzevj0~?c)2td_+7t}T>N2Scs2Q2ybiC&8}Vkm6>rBo@ou~qOW*O#*}6~sfAJCV zaqWQAY4P+)b^nAmLL)uPeu#4-uh#$S{`S=RaIbCo`G2n{2=~!Db{Kys(DzU8IpFzk zvW^Y?A@o;x{)aq&#I+h87B`$x-jm%(E7$wTA;dapLO!3|?|T!TKpgwYol{Qi@g1!4 zpQ-l!BR~Iz{MrB89{=8*A?vu$;B)u_zJxF1D;V3a?@v4X|GNIhtoHA`@o#eYF#ESw zJHh^G`Y8D#-_%#7^L50w>^I584*o3oHvPN!KDtZg&tlhCB!8ylH-W6UV?xlD+UWr%ZwRjy~ zkLdgVMzX$6KYVovZ>GmFp|_LojN4xk!n?`XSNUEt&VM>W9z$c~zul!Bac3BAEf0@- zUT+n55AH>J*8kg6<>5a1#(;q;tWS8=HSIF3B3J-~EpDPa!lO2(!^3YB8 zA>p`g>GWjE^}k6s##ioA{_YCB_DiBuxXWDr?gP6*4|f=4J&N%;5%EPyXzl-nVQ9Op>uDill$;WXCYbM-hn{$X?PoLbgD@@>8 z`bOmFS!CSz6xt8n6>boJBW}WGbR4)VB*Z_9-qYaO;1>F==%>fE8IsMO6>bwZ?48_B z-huR%UExLKE~Km*Frn_6yjT!kBK%V1*PT>eOsT6f=$t7JUFh~Md(M|@-+M3T%0u6U z^3cDDeSgvV9Mi`3&#;$`!N&P`uawS-UhU_pbzw|>Jic8Wy;c9$dHr7()y;^$K{*U4 zQwD40|K0LGM&#>Mx%#P${kx5wdbm8iT6(WVM_qY%9eI!MvE7)=K(x~htYv<93NO0`pB^v_dnp3XP;`-l1(vEA0rO7Nl6Xmb23!fvG{$*EK z`r@we1^SmT>^bM>Kf3?;?}0Ci`wCL@3D0OUrClKWb!1&zuG)1bT&HV%xwr{@6W_*n z@qIjs$BClCmV7Bvo=bjR#KQ&Q z<-)H-_Mmz}c&bMJBRkpUUE4ih*V*GbW1A|u`*Itrzh-@SwPRk3*WvYeBi@X+;_Vny zpNva$0@2rWij3)IOP!}!{ZkbCU$j5ZX@9DZM#z+KnjE!XJ3IfK(s?(sFO$EmOV@E) z`){r=yjNV$HhJT0VK_oRhJN9`E$U#hcZcsz+&#D#_u+~D3HJ*hS57=c=KcG=cwKmy z-hpoPq0`u3SBw7lvut~1O#Z*zr`7N1*QY&XpEM?r#b@w2d;wp=<^PZW)c=vc@09jU z2H6(v91NV&|Gww)800Vf|M+hCMLJ(WZr%0at7KYSs#5!z9@jLO*sEPa|90F*INKuc zG|5Bccg1}lkK!>beTx48c?oN#luJ+5_8=GjoBq*XEec2VSJxx<>1>Sg>ivaJn&1CP zef+PwCNzBXx^RQ|rI$S;+!(h%dri2BTztn(VKezG+=8V}b<%gVD?e#H$m`72Ke;B{ zDn229-$ve!4smyoFG8FH(e2ukRo+{LvQ0St)&5p#V zSerO8y)JxelQjfOOF{>_+Di2il&G7Z z$%jqfd()=SYs{>_;A!69rZ9N1SRZS#e$+#3^|ypn!P~U44}~siCNYQ<#*snv(|fqY zc~^_VCvj}+E#duzFAAq^JA=k9?d3A8^PHn$E+_&|P@R6CPgyX-f>`?#QU;InqW1ja%_Z$DKH2&xN`uNF>VaBnun8Q3C ztT(@p98))qXXG*R7mlCR6otXMqA;|VuLVZbSU<$;3CqA>$HEb3#;@= z|HPYE_V117nE&^y*6%BEpPQn8!*yW_1sTtD?z&J&Za|0e(IRpa5{~PZPEY2#P%K=6 zzDng9o2j?ux{ySvxGmU^W4Bm8img zq&?3##&D3HU;pEPa@sh{34IAq>Oa|Hjf6^Tu+EypcZM$=4mqZyPWfM37!K2WF6tj> zD-6~28uY6d`exMGWbZ;@s1-L{bxo)v>k;kh2J$4LojtKZy^mAEO~|TGbC^N~%bxK8 z@y%#KE837i7dnmI=lcgz`Uftka~+dJAHDya@%=9LJ%;wO?=e!vzDJrqim_Vr5B9i@ z-L7N5^U^c>_y>@=o&5hw)wj+wh!n=rQN{jN4`=8-`_#=iO+SNvZJRzM(TgYkfB3BJ z(>VS&>R)!(481WJul;O6n5EBQ9t${!^XS~K{%_U~jGl!4S#9&=ta3`*ydM|rvy2s7 z#3~B1;b#?wd{@QvV~azf~li zO~@X7W+*1pmA;>{HKBw)Su1Z=uL-5}Ey(${vbeH-qCpURB3^|v%T!ij$|Ui!9YhLr7B*54YZPfmM&$k0zCN6*GE zrZD9F4j+QIq%12^zPI-?BI9-GoW&WJxF&9j)s z3}!Khc`P9RU+;`@rULnh?9+DdKjqtP@eMa?|5lfVbM`xr3s^?Sq0*2zP#RX~J*SNQ z<05?({q#O0(Tjqd{6D)n6p|Z|Hf~--ZbHiV`9zy>e-sOsAe-16a$UymjkRa6TbhF? zwe9#i{W9!2iJ5{%9HhgChP1#jc~& zb&ylSnREOPF7Q7f+na6bJaq*(vF^X#e9%|L(vJ&j&*o#>k5NzpDLDjykU0 zyRMN=E$UE@jv4QNt~4~zdyMI?y=l!Gpkb1RBd87Sv%otR&eOl0p zHpKO}66A@$mUq9mE_Bh87(@!44chXzBr7L3jL2rV-yRoG5+3r^JV{<=U9K>4Qs=k zbQhiy!aTY3Vf&Eha2^-1jHOrH93pZ3trg*mSViF@zJX%pXU&?hfqrHEt>e#@cfX9+ zy~?P#Zf<@ZT{6GE|2g*m1?3nyvZ~x7udMGsHqZXAR{z%gV*7u1ZhhF{JZ0F4a-ul=Iu2U@r|z0?nBG&*{!WEtUTZ8r?%%iDwUI5}0YHWZweSNR7O zl8gFhHjqWwgkqGS6kD(r+prxwu=EP{L#6&ddAM5}xd%Tx{*kp#Seboyq8z&s*D8*A z>L|uM!*S%FzjrX@xnF($^Z|MO|BQcl{=SEl?;+p*a@@jOv_F2p{*UYb{LptU{k_pJ#4&WfN#{LhHOK;Quc#itJYfU&T-1ruL0oD<$rgzloS49my&S~tfSQBdLb%<-q z^(|{-lD)g#e*^pf6g!#B|NoDkWc&A}*vV}63dhu=0Vi<^O=w07#*)VO?K|O^Npi}V zai)%qPUhJ616}NY@k6!`lOy(zYy0Q@|H=6OzgFqAA%QNW%k5L>I_Q(^`H92Qpr??l zFgALK{eQr@$e4HI|NrviK=h7U*JHm762{lNx4Ew}wf z{|Wmg?ML>y&ZK^WzIppi`|p50gA;Em2xlC77Sou)Eaos)>-}S5TK+rZ{m*;ry*0@^)KlEZ>49A^C!rOas3mRN-Faj z)CH^h(O1^EN%vpr{&75*zwa4jpY9pFi+{k6*M-G9UBBZO%>PT0gGgZ<8Jxx$oW(R| zFpD`X`5!=59$$EuZ$voG|63r>;XE#&eR)k-CRcC~tLQi{|1Ycw1%K=P*U0}Uq;J60 z_M85{UgcfqHuVp>%YWMctUo=D?f2EP6VN8T*vEFIf1pSj$9)f*$YPA0Rlg(F(UX{> zXOLBA<}k2(*JIQh+8r;`O_l4cUd(cPkM=z4d_rD+bY5iL#{-F57?w{;$ zT_5&3W*;h0i7M>J0UX5Giu>1AnIL~|{owllhySB`f7JCKlExK(!Nc_EBUw$>pcc8E z?std#CF6L-mGO)n-b{B<0i@q1RZOZeV+AFp#eQkXxA~&HJB`C$x^UZ;M z-jT3itbdDeTpMj`l|9VWKiZT{cgm&^B6ZNwGjyYhaZ`tC4@ zAq?-iJB;l1j`rOh(&cxD(K2za>#%gHQG;5fn_Xkwkx)mUOu0AI(;JXG>E0ULW0QML z90@1IwU-_Vr^qIB%p3{LWD62IJ=bN=6|KT;=sWGXPJ7OtO>)-r(S{t899Zx|!= z)Ud`2~5_hzYeOu4!b`x=UYukvkOVo|E>CkOko@ubZTpKq1&_QIp_a^ zo=uYML;q|Dr|t7Z|Kc;kXEBZJYxEl*RhNBlO_&v)!#oyn4q0RZJ zj1^qODhkf{{;>f?SbF40*hChi1abavDY*q(u?<(||8A#G%4ZYm`|Puo@xoW<|E{q0 zk#p+b1!XhEnXtFRwS-}>cnfE=za2nWeSIE-q(Qm1}Gdi4G69+&T%V^t+M5X#~ zpL)}=aop@QIf(X!g79$5y3kg#E_`xXzocWv5$ov5GwP^$brqRKZdv_Trv5Ki|C7T- z>i=SOd#U=L9Nl2w^cCqZnvv%UrIT8Cy1tl_uz_BLO(;gbp4;I2SzQ}SgxiY?Ln+z# z*X&1a)vff7t=hb0g<%`L=a4>>1BGEbeFyrhMy_G5Pe z`xsaK)7h6-{r|VJZ`rZKL0baud>tjiHz{uhW<6{}UTLUv!tc&SKZK z#dV<%{n+i8J=lwVs6ZvEupeXEaO1P;820=mIVGH_P!}MF0ci|kXsh?*J&tViY@H|V ze53j&4oKr54xwFnbC^sdir5hT^Y=AAb^7mX#C4xn7NM8VK+-<|`_MnnpJk8dzE64L z`M3MtYUA;{*M<(%(evy71oiNC_3&2tt6csf`_Nx(&4DU5K!fe|w&Q2>U%fOMa1y7` zgl4p$6=Mh4_lLC+$jQ3PbN({IS>fEg_FsYxkkmgw4r?!s&{M7InP&Y1`^9Zo6WXMc zKo^oo7s;a-r03VyWhbZTJ3~gBBCh!u`vsyM zJu7Yw^N9Wjab3uTkMjS2A^88twFl1Gb{-e7j1^qODhfXD+s2Z9;iVs49~$4FeYQ^d z)~v4oqVlbVO<{Wx()3MaG2+_O6IH&A6>Z49z6~f2u^ijBJs5x`6#ZtNzy}7$S%5H&SbS9&vr1(F4Zc zZ7Y+`PLyLeI?9!E*h7zV;-dfiUiv=tXY{`#iQa9Q$O8duoQWg6DFp3&!w7bVzvJM^EQq`V$Jv~A1E_PkjYp`)qfUk^mFp(DjWTR{Hbl8 zL!oUOP=rk=MhQx>1)a)-E?g}i=t=aU9|ODP?|t(39(lP^{;u`@kuH(HF}6+lfQg;X zk12Y7eInbUFHe5lXOpsP%jIo!-+Q;|NowOy!x9<>bk zVc~tg58?O^pgOW14LFHm?eSA&6PnS2=-ZiE;cG^pZ1Zi-sSm#OOnm~%uOvO892oSi zvIjbiiFcJ4Kip>g5c&E2Ta6zY)9xpCIOdrDA=l%maql+aE;RZ-(Ft=Y$4(nx#{?!Z zg$%ODVPL=UOX&qN^TavoqiAn4UMamnq%e+-#O9DlNrxWC#JgKIhtu>k z=)d6F&N*h;wO-sD&Wamuxj9UeGe~D{4zuJOQmLE6gl&_U7hXWN*0?{WkU^(w>Z)*# zO80+AnzhnKAH5&v9CIEQu#6R4#3~BDApg&~f7dkOS|%}d-gP5u>_5jBV4%ePV@TXE zM(mS9nm&s5`O9k<BKlrr+4|e)OPzcl$Q{T% zX9pYq=1@joc;V)-lPt$>?7?2_L;Ey8G%}%WSxHu*L)?Dy0OG%O;=h2B+Qq%-m!IN4 zcDl4dJNY2=6xhDOcCxp`c5UiD`ti3FhJ%hjgu|%Du~v37j$&MyGKTy*VPpnV$RUft zE!yCFJ)^DK;7D!bgD-BhSQ{LjuCZ&zxzSUkTrFlJ7qX2D*vP1HzFO3w0rB5~C&^Q2 z!kGQXOFidpo-=;-{|@=}*DLv7?dNMnj(8ubGUb1{^564sG^g;y=L$8miC=`+qT@BIo-&bYQ|_B;Iy za)Z8sv$maftz`RLVW<~>7SotPN1^(+bbXkm_o#cj+3j=mdBlI(^|9NN>fT;*LEIDj zkIo67#|11Szy9}U)eH2CSVh6#Yr|jz^8S77{BHE1SKA?3sE*fu=qHQpvkAp0K`FLi zE4E>5yY|Pl{;@Oq$F%pR=vUYOYSsSRtFO9}e_#b)Ep`9Z^@+D?e`p5`t7ErIX9vo# z6VZo1)wDj8(AZ}1HzWB*q}3>%L+8Nt@ z@{0dkEB%W9TN^!rpY?z15|_jv+E?XIGI7EDKep;PJ%iKeRu;rEq8G7@h-E`Rp6Fk2 z#(rlpjqGc#3$tY7Bkb#E^XZ|_V*%%I9v2YD9=hc7ZuFp6-cKqA^6M|}jP-9(Shn8^ zE@Bl0U-bQA1ID(?|MJ__@n88Wzy6DN8T~x+>n~Tx|MLFuHu+zgsjc!qMvK|@d=87G zv&ffl6IqNn2cv{6#qqaYAGX9e{|VblZo_sgX$$NiudXM*-Pq3;*${8go)#b5A3NiI z#tO>G{QnR54(y@t#XeM^o$Wc}o_5nKQH72ohX%)- z#3?kP87*i<8^+8rA2$bKqSAQ|Ntf?I#$1Ff--Fx%{s(8xTgT9xz6OlYQ%Kt{{{Jr_ zjW~}zjV}68ZNdEd{Kkcbi=|g8J?UhTb4=1cgP5>wa)bW|B&8>u_y764Z-L$+E`bca zXP!OF>>o_{{@Q$h7@?<%w5j|*5vx==n;H?Ghp59l{iFJGju zB6mo8S6rrs{fmNW-w+@CLUIE-cJn7DHz84>Ufn0(s&|WpOOT&ew_QF~ZX{8Efqjgn ze>4YE`=``CThaJJcI~UyhHdoi*nu*{b-#C#i{{Ojle@78dokRk@19JJYya1+4HfiC zR3WZ;HHjEEwkG2I;8n+G-=j^6f-k89@c;F{t&smPgt%7B&-&lSe-dtxPUCab>t8Df zMfCKbZ)DH!f!Q6tu?F87wz#JDn)RX7wk_ybF-A;oLtI0kdq%y6 z?ZP|I=evvRTIByTdb%K#iQ9>C?8YAK#XeM^5>?ob12~97IE?H&tiK!X3=5w%PW`*a z%076FoqUUk-{g`$u!i5Y{i)g%_Fw#$=Z3~tm4_3?*lLBBK4soWlrL5ekPSGAQ)ogn zj^BA_XdzqCh6EOkM|P2M{Cl`Ue*k?DDU2gs8~cFR0`zF}PwdbqPCtVy{azV;k5jS# zv0Pg9@_&o`-y~0H59I&=i!`q)7nhKv2?kjpu_{iLI z!bcmnhvu^9h5OGuFMKSuBm7Ol^FvGV^TPwpW#Qx7UJxEU{DRQhQWhR6e_{AU*$YFp zYe>)TGUn(y>BBR-LW%Q~VheI}yF%9XjO|mHq)(jR6}H;;q5aPeUFSCFgLq2lDb!D1 zq+i_JwqE~i>)&mhEA!OQw!qiHzU>9;!o&1LtNC%w>%ynUpOPOZ`{c^b&{k;uf%A7- z3#~kCx9<*=VJ8MpKP?OxJIjxg)jchwW{r!X+_v2~wqYo>m zmuw3SxTj!y_`t>OVV`Z4xVKkZ8U$G|_wL-Q#zLW_kkpJ# ze_)IC2gycUjFh){_=ffzcJaNrqu){Hp zPt{laH+(GUJ29qy9*=djx+ePSPZg@0i!S@D<v(;GPwHx zfh7B1zrqlP8`%G?#vim((%LJd)#9`Ts-&|Y2XGK+*OYSYhv<{))(O{om|l&zreU_q zJyy6^ZH*dn?b;)?WF6wa1MA5KBplnV?GfkopA}f1Le~tN;Hh zS2l0;Znvwa$tK4>tm-9tJ+FgkM0E_?rn%v_3zfCeeLyv2F zXVAW&eH-`1I5LQ~d;Is;X?l5cy z3H|zW`ov|QR}`ilH-lNsVIB)Ohq(6YSgm^}C$!V@{aSPK*co*&20fbr4Da>~_PTd+ zq{_XMY2owIxPWD>;3AgH!(Sx}X56nawnhKkDx@#GZ=KAgzTO}|Ip+p(MHoJ$eMTPt ztmjv%|CL^fRIzK?pk08*@AwY@vh$a(E$=5jtNmubt=NX`So)tK>>$gc{tEe9AN#_H zcCP-HMe|B`iZ91*?7?2_L;E>>6^|HqZYc;A!j#*nO1epL9j_WH)i zO|~7tK^($iRHFv9=sd(OW+!)}hfUj?VgqNcS3X$ht&4rlrXE~jU$4fw{Os!s?BQni zbSwM%q;=)k@Z$~a^Hb42_kDV98D!}>)VHb+QRiH-ja^SR;3Q6=F^)}XFE-Je(SlaA zMZBZo7PdCM3rP$jz27xN=#v%BUFRC;8RT}b<@d79cVF&2%Auh5|m;K#?H$Bn3$6X z_-~lB^Nn6qCrD!(wqv--++Fg@`m|;AohZkIKDp>aGPSDSBD08d**ccjgx$99LG(G_ zOSYd~6ZVl6=*y_9PHAf<)Lr!Mly){*XEMC1d~k z(H*|Ya{0ZKJ@Z3$`r9^!>A27Hd`EHpfSqiF>-hSA$Mw9SFnmoP=#~GM&f0I%Gm341 zIr=)ql3x$G6@3fxI^voiNp$b_{ISip?dWV#|DwA~J%!%!%l*=Q^nUEH&lBsPGU1&l zNA_LXUrF_UN*$@+V3M2?&gg5;B8P!C_CJO&j1k0nap{Em-+2$Xx7Y^|*N>f{r_1%V zOM4GuA3(JIC(-y6_A&b>OV44d+IbLd|GoCzhYD1pqt^I^ZB_IhX~#Z*{qzHf|J3P2 zQrdB?-TeIf!}dAo{!fW>P00rP(EG`QjyZ(Gs74KHQHOer*=O9o6OOz3|AVA9Lc(>{ zvHw=x{~7l`?f%J;dG}AIg-6NZ3&!@Pa}uY}gmj^OHmwiM^!)tCo#v&}Tk*5=AKS#W zdrt|n3mv;%KY84DpQvroD+#P{uYU&6g)()P|4_U~!m1$xknr2N|_U-zTl zSp89qJNDQ`ZG;b-fA76#gimAajBV3wY;tm5yMUfC#-Cl$PH6q<_`EUtAq-;#@!ub5 zjAD4Nwt;lcVj43@i%VgaKG`5`%+cqOqi2z6b}lT4YcJw6L!L(m{Q|j+1U>%qrw1#- z7m;6ANVs=K8w9aVUC<8^*V$i0>^EPv-{L=pQ1CzHIc&hv3(cR6@`vh7vN-BL;(wwp zi(|Jjj~oxBp+tNsvQI4yTgk>pJg?r;u#LVQJ5Yw5D97X(evAAP$N40V@kh)}^IJsb z?BzNB!==6*3}OJI_DyS_r!a!JCRaD&-2P7V*ZFplL@#zb#~$p(K2)F*RoIWt3)c5H zHs8I&_&4(F;^g8L(qXC1SeY(^>I4L~2LE1Yvhg0;pet)jc z*gEp-|8FvOt_{(?YM%2R@1cVKUzP3Zg(mwa=-u}1*|+)f__y-BPg?!xHQt|J|9`(R z)x-8XXg~Y*l1bq{vR}H*(rG~}+G3jMLK1_xI{&s-I}cOZf*Iu7d9(6=gZn?_{>kB1 z_fMvT)8weO*zg5m>5L)W0tXk>>G>&kgF)1l`k-M zQvSz;ygnK8`{O)6X4@OzqW;xZETwlWm=_^_3%%!%dpp3d znZ6DERqjn(61~-H!*+2yP==i-$HJXAhu!4ji>>8A?!`XD@%Rd|5>?ob130$TJ#5le zPzD?nUi{YDaEOfUh{NQPHvI87X)|db*jmaGHJwxm9&2;uv}SSA5?7R=>o5 z-1RgmGg8Wq8D+>>Wyxv2XpT9HY0MzspD^fsVODsI58pT@Fv+H#I>`Pu29PClrRx8a z$}SAC&414S&pz$yxjE^~Bl@8&kR275{r}F=ZP?skpr{>&cNh#cY?K%KS9PF$HMn6Ljmi!1X43%=r8#|9MPf0K6i z;aOdEqW4$sJ1K;cw%B4zOG;^rZQ5dsEw#|5Eyto|lu^c_qB6=Tql}_5Hdt&^?mz;h z5FkJb0Rp5UkdTBVo=%JoB%YS+eePo{=X>Bn#ORyBn&~@L~kVURQHd;?e|2TR^ z`c$5FT#!D|RV;m?ZuVcz&s(VcXKtD+eUmNRHOyAW;7=a%QGn*T%6}Bn+w=LI6Awl7 zVstZiF?W(3h4E0rt}MwMUa}kqQhg6u6Pf9Uj?342yaRI+4l~y>SId8D$dUcMqw-t& z%?IE7TCVI0O|0@5)VQxNSRmv@!@!TFWJ+&@u$BST)3?tQhFrm`Dpf1I@c z+!yTwXneo#$Nv^@9#Yr4U~I?r_A}D|DRJPp=Abi2b<(aVZ$NZbO~sJ?({Kn?sGg>6 zi^HhJG4DIgeVLer*_ea5n1|N0=J(^Mws3o%dGXrKo!Z7-=x%a8z{$bRh+B251YtZMA zMXo^eS#u4@95m67>a%FqS253bq^M3Q&k5 zM0xy3|4*m6F3hbnd_Rt!@coF^By=J=GHcROqS!p+7PuF@~o>-$gn{sG_5zLJ}UbIJg+9wT!<$u>m$ ze2$DuFZI8fkD-g+z0kjp+Mx6bbNDOG>pwJE`bRZt5bg0iF|76{O{=sv`zsZJSbMNR)z52=A+rn+Mf5oQbSBKO5 zK7(EyDAiVH-$x(Wf7E*n(9fb)ANb)6?}%s)!JaAha1<81&yEQt>Xiy&gEg-{C9++bMFXSPF-u9>)Non>FQ9Z4wzdxDdbI=6xLvE{pI1D zXAEK0f@{L+dDnyuv)>U4%C8I?v8nA!neZz~=Usnl#^ug}=dV2@i^QX+ms}dQ_g)$< zxHf1%|1Bpj4qLM?4!h-Z{f}9{!oF{Z{Q`gPUmHK(Deu}PKkQ3Z9kS z6qD&w@OJuFH;v}1mwK;p>@zSG(=Z(~kckRyjLHGy1AW>WsHTtX*L%wN0P3{A>v04P zXjH~EAzH)Va?Jj}iXY%0(VM9~H-y|EE0z(OJLM z!~9OyH#%OwNt&^LZVIr*+UvCw^~ckTP>d41eg00q`ae2~r7LvNyHO{d)yr#-kPT<` zzsZ-Qyt$d)qF<(Ap*iI!$LU$&=(TR>yW6$iO^#h&c zF&sE!uM+YE%BT6R^ZFHstw~`%g^_b#R_G5JkY|t;7nGM6S)Zf4ub!e@oT`pYMrUuH zcF!5~q7Q#v{~KU#EmHoMD5Fc2|L9nzZiy~*qwcJ-7e~%3>y^EYy~_V4<$t~MA6+No z;Vgg7;XE#27)fWOf!`VnBcndNRI>jz^CQS{7}-yF;;ZrYxt@*j>?WXl96wj6A1w1O zv)*`S?(?t4(_Nd4DQH&iHz9+*Kk3zY`IOh=2hO}6KbZ9A_*8bKqAuyhc>Vmr z_>sAT@rGr8h)?6+bj&~|W??qwU@k`X|Gg_&zoP#A=PwHL*v-cREJR|u_?s#Il8c#_ zU@4-u!Qi*_f5}&uF|YBBYo8d7XVDMHW1~5gE9g~A{aa+yb1*WmdV&0No^M-Z{!hVc z@yept;)hCJi&v4=^{>TiI$w()u6!+Ci$(6qLp};nh$0lD1g*lZZR+dsqhx#L>+ue< zlevrR?s+|4$B%j($$2f_fJQXszZP#^@_M`_{q=Y$Kl-lo56J$SG-zPq6Pn*MsV3WOKG>tavScn%+M3wfIqt%+IS9C(~Ywccr`* z?;QVHyaQ!LugA}D*NZ+3;6Uf=@$$CU<7eqr=U$H=!a4d7YJJn;v9sKOfdZUgP^)=VY`|Eq;KV(C%{D*8}j^>EB zWSjrpG#sDCpXrGDU}umAP7KG(kx8#ozC?3xXVbfkkJmCsb8l!0=>ZSsesWcxyS0@=yjMRs2>pGIEOkR}az zRu-bNa|M}=Xbs>|X{25H$zjezwDzxip?Xf1dJa9GT>%PFgknVPhY~Vc3sg#$qY{J4 zoss#vZ`R*y*p1Zt%%xs?mA--7l0y@_HgrbrE416mzDHk+A1C`?64%KdoWg0GK`;8y zP?!`3$hX#)ou!|{5RUrS?czhnQt@M0{(N3qIII1iq5PY#oWMx`cad_ZL>Ws)`-IlJ z_dG6O7)ierw~>l8jLiQ_(Y98Wx04;({GH^;_3D1Y!DRZNHI7rrwfbY%s1KKU*9_*Vn1<;%piC{t z40_cH-?mIUnLZ2Ap23GNm}4+xt%3SVMWOGEX6W3 zPf>@OW^MsJ8v8iP%?f%px|iz9U;M_{N5=|t4cL|W);uyF2e>aFqdka6<{!8gje&HD zlSS+gPxf8@d3C1mobKC?D+B7){mDa!+WysK?Rn`O#qKFVDauia8q}i+QQN=un6!09 z8Y82&e+Rp$?cX&dZAxcRA3(JCezZm}TAv>E0kk0M14zuV1`n&G&8RP+>sfnmlg()3 z|H!$G`qQF4K%%~Y4*qsVe$%7AfHL_|CwIqi948R<0Yq*89(vVrc?V9>PotI|^#MeC zj7EI`XV~?k53|JYLF0D=^s_jJe*GpxWZ#CQaGtz?VWd1O4Ir^p`JAOZ%GQ5i9^;yq z^anha7SdcBhw+$zbWFw+WMC?$VLE0Y6SFWIb1)Z!C-wims7-Glpn1&wSJ{JvT!4jG zgoXiYLCD)18 zsBKM-?2|_(xo4Ge{hWbI1@Xf@&FIMff3vl-=W1t@9q6QYk=-lI4_K)E zdqMkGd$yrm`@d8BmyFigw-jm5@N3XF=8^d*Kp~1yj4t6{LYAT&mFPE*qK53V=S0~6 z|LIL=LnjWfD?hLAf*!3YIHWy(oPGk)n)<^t-st0~Cf`1PZ=UbR+xr_V)Bk}U_ng9M zoWWoBuk|vwp3?ruQM7w*N1ygTy3mcfao!I{(16Av-;ZXrpuxKh@MmQHZ>#wz^ypgD zM{R1Mk4#0mu#WmpqB#s{%;Ru)*&E*# z^_@g>7~XFGtLs)3Nc+X|036Pj2Ov6sZoK;@AliRBot%s*$iUm}fA_Q(Di_KsdN`ri*u~++_Y(A#_&s{^4HMabjhUu7r<_X-WTV&GPz1LC9qR&QjrgoR> zovue`ZO>sB?SVI!oR8@2+Xdu8l+Tx^t?*q~^RP13w?uo2)yi9{$r{)HZ|8q4;?H6% z!BRwL0VayI1LXm$&PQ{P z27fBOD-W}y=R{`h|E?GH|G((`3g-Ta`lHCGJyZ~x?ZFuBKU_#JLNQ7Zt!r$`(kDuP zYyaVLdL_Egc(+sDk(&-gXTHv3UxRuy;efe%<>vpj(MR_GA2J?EKZaUmQ8b6XhMQ_6 z=E?sS$p04Ui%@4c&dpov?@!Q2*8e>s|30UFpzQ5Hv~N$>DeeCr`}%Nm3a4=fz39W> zgUMka((lm6Pxfz14(G@roW}(WBWn95{Yrj}RHWg+u>E^6j$UQl;gGtHBb8= zbpy)(bISi=<-a=6NdL%j<^M6|KX(=WMWugoC{;Th)f2SKahP6<+|T-7Pw0E7P~TAo zP4~lM?CKjILoP+e#aAP%F}yuB6NWFrT8$VUMRQH0h`^Pd)* z|D+slCp(nGon#ktH(95yYu_sQkN9bM3L=j^Q|>y&a-GUb~((CNAuct}vg7 zeGf*;-%RNb9o+PNZB#hLzJIj!F67|J5YCXj=!@*F_Z*1yA18&gb!HjO_pCn`=pJwlS%rBc1j_5yaI5A27J}EREnu-n+SngGEi5X{4`?$#0A2L%QukP+pYA!6F$EcDKE;o6YtZTK(Y@#GzfYfzZqMnO z!H-OSaW{iqCT3wa=AiGhlfqoGzj0!iN6yCr42JZukX(etSc07|^G=|PyVy>{8FaDt6g#LSOnuKdJJ)gd2pL$ZV zIG{{uukp{D{Ci>FMMiruR1{19IFu&+r}!u7{tbB;wbSME{3t{bicx}6l%o=@Q>6c? z{GY*pbY$=!UA^M}Y4Lxq^v}%^ZW_qOOmUNJo-F-G&*Dctnh>2K)kc;VOV{*H`Y{~G zA!)rz+O0mX+#G(RZBWa6!nGcpLR9}fO`bt7`Y_U8D1JrrV+NSd;vCjCnoIO-@<{*R zdFBfk#^Cej!9OQIM=JVXH3y!I;{P~uJSHF=lhLrq*glzoshEbh*8fbWM{WM2m_g4( zcb~jLp77TAcU_zE8Anou4;m*3pA_MP7Gc#&j-0QyRQN6yzKet}-dbOo_vuk#7Jp}B z4yH*@bIEy_kBVvfFJ{R1P6$6#pAmjIOs_?>XHQ?^+OUAz{*G%yw8z52NEdIqE=vxJ z=}Ry;Ry-w_Aqy)ayGO4z|IPZps0^MJL==78?B81dlS8js=-C)K|K~n^_I={ku>X!~ zROD#`7HI#HRmI9bWm&W@?%_`D#Cr4d_>qTv6rd1AC`Jidm62^V%D-~s->J&?3Ceej z?EjOZ{eMdNha+gf|9U=Pv?pLGf67sb8Z@VB|0cPQ-aek+Xri~Fdxf%fnKCv-S&L3~ zefyHbF|z;D>LY2wV!HU2Y3!EW3G^WMv)YVbmL^`3MmOsxWH-3O{{ID|Lhfi~YtU}n zU`_-31NrI?IitcEdR39S0eb0usMU9HSl>ZSp85j@*fpHJD4ZqFp_x8Jo<|e?t^U6Y z%)|IE{eRlyl^9v4m?`|H2>+?Ve~$2DtgfOL%X|ILy97l@zAosJda zC%VMrk@NqK|G)JAaW|PCQ;>l&bIYcZ<>UcOqi>r0`}lP7P@4FgBL1d}zvN-&T5`KF zwWk*wTbn&DY%6(Z*t+bUVaxpS_E&r%zInm#<9iof6!uI{2_>_y4!f6J9d^y1V2sXu zmIc>_9kZ3KpABIK|NB35f5;@u>SN)V)5@i$QDI-cIFNKts7t+H-Ew9)QZUol!Mj3Z z(v8*@&J105j1841?hX~l?haMfsU7Njul|cWpe??d`3MbC8F8G?%FV7so;Yy**$1sZ$$)UWAeT%jU`37pTh>s{d!Cg<|$4C`CCA zEVPbbfqnsc)f{8P+0mYF;xlTC?fq8BOjgfJ`=9IoQ_ZE0+>y;E^aIdG>i?tV!OZQ) z^$#3lk1lj$7B}@chT}Ma9-P8yRLnNUon!rh`>Rlmnkm*Fct$N*x5)Z~rPd$lYiK~@ z67&AioNoQWc>bW)hPe1QT~hXhwwj@|7ps9 zabitE-Cw`Y6z(%H71J;sGtip%SL=UD)PI>f3w$5h-T7wyUzvAgK>b9y*LY6-w^#jF z-Jr$wsQsA9pIMlVIcUxke~PrJ>Cv3KXz%-Z^!XS$|1Lux9HKQqT_=-6nKZV*wS_ov zOnZP_jPg^;o?-chGI|N~QXF2Y-(Gnft^KK?SMaA2hi0h%qnciW!zJqfMIkJ6PZm}n z8#!22Da{DC1X_!Ie~tQox%&T7-=F3Cvwc6=&0Obu>(BdsG#vN+zTG)*#s_@A_CVib z>fqYdQ9Zl=KJ9G&=cDV2wjSx%M&r^J7y&F}(!QUOj^kYkv=EXET?h5;cg< z;BD$tub@YJ!W~_rPC?JzmmIohNaqXW(NSKl9?=%rZ%7WEkqq~QW8`t1Ko1Uye^DH) zXp;`=#l2zuH+|xs{-K)lZ}zJ#*3YDmr@majlK!q1=0>t9OW)DLs9!2Aw32P}qkgG0 z{r+j@{-uS^^JyVE|FzI_PT@4p;DEWvQ5n!nkJhdo$}lgQK7i;FuS;@VUkT=yT9rss6#Plk8}+?+?3q7#Cn--GsvrXM(C{Nj|o-sn}r_)w32etI6FzWb;TzlMzZD)QMi zbZRG%g=ki$7m>wiq92{FUa-)36mu!w>KBOippVYMs92_cibJTve>wlA+FDrmRiXy< zXhIu0(YnO=8jhywpGQZ!ydg!tkf#5StkeE~`}`a3-rk>&-XdQ(#*gDTfii829=J9l)AD-zVQRw}5^QLpYEA zFOLe7#mT-eT6@dy)PVRKQh0-c+ClVvvu)J=kE(W^mc3KkJejDPCt$A1>S9*XK>Sj zig~^tBj3M>zo^06=YQ=>3TNEci#`nCEY4vFt;)hSL}g4n-md@CyHPj%SM`78Wn-JN z2+i~sb|Wk*j#cecJibgoc;9A{q%c4eiR zVLUkj2g)-;Iyo8THR2ApRdemB$((`1{EE(?sh%$#XU2Stw2LTz?D|$bM00+oa<^8W z@tXURmCO1(nfv~1?JJpyS(uGEn2ULsk7)m&1>{@%|16|0!eab?-Tx=re{hNWmtq;R zumaKAzijgV_x?W*J`noVCHhBO$N%N~Lk>UlkdFcsq6oz(L6`h1+81PS_x+)qd1U{e zCBkr#e1NQE_tySDHT1Xl|M_0B{V9^Qi?siTwEt)O2dF65{#2HY?0?gyOrh7{urjC? zrTnNz6WY*;V>pfzXkBLhKSs{?>(%~0uKll1y^HJ~*8b;Ly)y0y{>T2e%HMg)Uqt)g z^zi2tPU8%kGnBu|xL$huGUe}5^$+?0y0yi-l+B$Bl)sDAKiEb4-<%_daDdzMQ zn2agNz*O`9n%*8y+$6#Y}SU@gBg*N}&^Dip($w-?uI83ia zG=F~)cZ;zEOR)@DSb^51(!X}XQEkX}bkIA~665Xqp>nlf z%L>#d*{%7r{*-9F#*n@O_2C-z;c7C<3!**2inu982})6pO7sis8nREmS5HQJ12vIt z=)^G`#|iYHVaV7%InDn)O`bvXS@R6~;-Qz`ZXVE4^w9^^GiWsDe&X0Lvj4w)Y-Iocv%=7HC&2(XPPZV(knJd9PXS zn~gb`i+Pxj1!&aCVMx2=w023ab_&_j!|s?p z{G<9?ll%({`5CPZXwt^%(k5%4%5UU;J{}gi);)p!IOE?_JS)Q*0d{4cwPu}kYLAL& z?)(yVOHp2~?Y_i!VHtB44$t>p+CtS!eWx~lL%u!N*k>aLd1%h@eD?YD_9D+OxFi(N z3((PWZbmAC}qasPW4sV~YK(E1J zL~9)qy~5_r_;-T)9^~F9ecRXZG`%%l_zRz-Q-uFn>Hk7huXml^jk-C)f1dD10~)9M zHvTq~E$%DJPYY-Gv&PtXbpA&#{eUw`%GvkPtBUmh6xbt>eipSC^#52VQj@3u2j|!| zlt}vzOIHsjhvxb5FvRXWM)nU`rd?njzy;=EbWPJPK*v035Osy-2o;+fR4NZb<0)wu z&4|wEYCW%gaX}jH;m>jR^}3JjV(uoBo|k5jiZtXsF(Hg2$0JAFt(Yu*OLx&8Sa0`V z4rxE278AHj$7D>w;79B?MNY*u^yh{!ogArukR$W=3X1f_`PkhkZFA0^_)bpB?FA7o{UF=%I_R4Eq^ zDGzI+K4?T||3quWXSp{j8>2NxQQN+pxe_&~N87AXAsPc1Szo8Wb7Xy;zR$PT*F|Ul zMP~#@XXGA9Qm(6GG}asYt}zBqmK9q2zrpy#&C;*>UlTtMa8vGD8@=j;_Zn8-(~n`K z|6!T;T;V-WnQy?Zq1ayaWDlZx#VPVMM$Qk?){pijJj2|Jk#h-WtEaH<)CbVVZUAR- z4nsJP3mB|O3BzRHT`3{ytnhcHV+xsyG>k)8y7Tv@zEOvW!e>0Y1E;;eI>!Wh)sXi; ztBps`y;+%lOt|)X|32@}{(rmvz`r=m-&(TkMSXwnU1N-5?RDZNeJZA5I$E=ZlYe+L z%|9SJgmGu8`)1gG0Cnm^^;yE6y9TmxfqyVt*!#aNWJRI$Q7n#@N*|~`C2sbLqo>~( zpPJzrnV5y{+pLEr=U^`8p;dZsTPS^M&$lm@KDFsP(S>e|jIR|a|MHZ7`uiHQ{Nrrt zn`~j8&yNLYkWLqpP0~*8I(a?4S^78?7{3*R$daN{Tecwzg(dVjP@Wu)am=l>KflKT^yD^qVd{i68+6VpShG}Tt1{zbM+TOIVb$9Grgf10NrgSYp898&+n$ocC7 z=J5YqUZUK~(SMmhF~3Vtiox%QM`WMAfl9Lf9%}^552%Uk#HD(&2_y6UgiY>)6V>sI z;j-(*F?28X4d^`W-SF0WpyRHcK$$bd3&|dw!f6~ZCSG2i9?sCK#vA`eFTD@7P1lCQ zs7W#YjRAINaSlWH>+!eq%oi|>Ci&$TuHi?cF#fqZ5c8LqD~$hFj{mFq1=GwgFt&f# zoPyfPlk|`L5BFdFU+(`pbL%Pnzi0IS>f>%lN1^^-bfFt{{HW*W5j3DN)!YKSb-vz+ z|IWD?d?Y;xitvr-86BSaWAB(@d|-<6&I--r9-kier5UeETpJ$eroV&!9dY`!wff5E z=$lr)d2h3Q1EM|ttDiF`PFNi+XZF3TUGFZiejlCeI#4#=nD1N98yCW+B~!wIOyNJ{ z+OU@Usx0BZn42gZ^i|i+75=DU7wuKBtM`hqb6|WZPP#Je=({ZJT{bD~Sur7$T$m7c zCtVY^4onSO#@!wYmu7^`*nVMLczS4DD9XFd+O?_X0_X$ZrVnlCU14>>Eg`q)mXOzX zTUe9+?yz>nyF<0-uJ;~K&A&aY&v=J(&#Z@=aCz7`bVb;d{?3u_+T{9{`B#Rmg;#}b z%dQSZ7p@LZUzixSXWO6Qxp>%pvGK3@-qnBFEuI#QRrf(_{+Ljml&mfxzstEOoU~8q zNo|Jjxwe&`&yoH3cck(or~Ey!Q}oH=)6OE}Y4dN0brj0)>ZAWHzf-2P%2V51JIcJB zAC2_nj4@>m@l6jAOH@=IzwXZsBF= zloy3EQg=YYa_B_cgj-|(O13Y!B6j^~^&ju>0NKpHzb9wN3XRD}-I|H-ledyLaFa=X zhI}8nhZ)I6S=r0|sS==BAo!Up{h1?5TM!~EJrZtgP| z^xUYh#X%EWec(23isoG;jU|Qc+TA-&sfQ_dcPf8($$xh*alTve6|p_s?8UyUD`KT> z7lmi$Sc{W4HMW1Ue#fc$A=w{nQa+cCi&f6MA~te9(e=KgV|uLTzUeWqpE#uHUX5C` zxYmW!corvIzZ9*d(t>~40!2sSd#U?gtB$u%2-F5 zvNlP%-Di(0)M?k(n=9IE{bviejpyw_bI!WZOm)X&&e0<)r`tEO#~c&as>zz;Dd8FR zWhp6PpJ$ZLFs~pzB^+elPnM6j-;Hws8W%wdKaYw9YL!?iO^;|J6cJil@ygPRCYuCkketc8x9gLjM_4=H>xAfd)A(&*LFJ3uK3-tn)0#X zUzn?oUmuIR{vFqEAb(8i7E5H%7osfbs@OhuSMxU=dl!xkd*+QbKV)pUiFwz8$+073 zBU;doTcFD>@e^#F`mWf2C%@`n|1&nFj13!?$)^{O4NowypM72IbKXCH`t7lQB>xF3 z*?*l}4qaV|zeoc)o|{;NApKiMv??Tr*VyM?CjDPwzp>wr4r}j=g>|>3g#7&};i>Rg zhzZXPn=c9%k(Xc$TID5$-+4T2dH9=Q>xWi^cd|Qa4%^c&rG{1++2(JhhHKevVLth1 zd3YmxeU8^h?j}li>ILtPbp76~>L2f+-;GQ>f{G2JL*;KD51*n}J&_zn+_`@-^GW6Q z60&~VN?>HH>FH+6_Ky-e1tIV=B%X1&rcF4Fi^l;l{?nvav+JQ^L$>;emFK=y& z%6j?he(5rCeXQpRX>OZzo}3Zuc~P4AJa_t5T8}9cPG1t%VSBwZF8z!r?o{Qp=<^J97FLD1# z>FVTl%7w*aLaDmaGvvt*D->9rBwnz9v^+oSdQ<)sDWLGzL4j*2x!#wke~7F5yJGI#EYjdaScoyqI=z z=tjb~Rx0BU)hC6ReCZnGO-K!Eic`bdqSUaiFE!*(nG$<9`&oDdU%)p}fXy{yLgAbg z^S@HU)a*>yoTZ$0f8&x=b0?M4 z+_$*Dm2B&q5sV96>N`?G>0_h9%f6*o*}9)z{-IIfa?@)j zV!b>^mwe(zta~^)Xm%!UgDPd>E@*Nk{vAig8p{cz!VC15(5!OR;_RUF^uORJy?wWS zmt6fWDuu7J+v}|KvXjXnnLnvm9g@Q(WS;Sq^6UI#`nnG#2VH%Mm%QJmsJxk~{-qLG zGuaqU#;EWccInEM_n}UI&L_!lU?YCwz88_=-fN*+k$4Cn$3LONx2@{EHMV--)>vX# zI=FCaY~;WD_qlyzw1dQZ;k_Q~kdKXdsUh3F8~UV!g4D33?bg`VSvSQt7vCByJbrO_ z+P)v#vu=uQJMK*1Q#ZzTGVeNbOKb;OykbK$kZo81KF4< z{K@78-v}+IvqJ0e3jIM@;pl~|(0*n`=%9D9?>dzgy3b@u6w5=bg(>ZKeZsXqjY&FUi`_}&Y|UD*W6FWcAr`vN)~-H?8*69Z13si z+CSN$wBVDmXQqEVR+hRv?4P_mlus4^@;@0nSp3OYMcXH1m08QpVgE+B-~aksd<>t& zKSJMYVh#4-863hT$m%|BP4h#5C~_ zRX=?qR6PGgsC@2;u#I^K_MrmL)IAaE$PWAmPChs%))VH$PQLm#+8rN^#iWUMVk`Ed z7VY>heuN_xPlN{ig#I)962C^{Lr;XJPduSh)cJ0CqrxhE=&O;Nul|cQo!Yx}lINDBLz%TH_n_9lgcWW_mS6z7%C!{*15)yy^Q54&DV z)^T6oq<_`@4P+y8(|rA_3*Q*u9vL6c(m$K6?~@$a|G@Pw*SpE>cM1=@fI+;BKjC%6 z?+S@4aVPG@<5-EcSdXpPj&^k8m-sdM@jJYNNz=81a1%a=_1J`3bmP1D9WK6GKMp>Q zFX022$&+tq94ug0Zf5PjCzgJreEqDQg_!IUNj}156qfZ5QB2i=g zy$*p@8b+(hT;{yf=HB$G8k|L}693;mUCbry_qe{7+*e~Cu_kl#JB>w>`uaAhi(PRCd>cQ*FYzb5j`$3586Uu>@ELpoTd^Gn@dNw}zr*VoHB)Ed|zRC?gd+@6-avPGvApZyNi}u8$KNJrG?9bvHhA{YvUqjSPaxO6=s?8Ygx(#45StK&RK&+yN`tjzeVu@GZ0duF(H6}O}HaR|ra zdHy-MXMs2nh3hBuZ;-1ViG{uX%hST3q=LI;%0T8l`kad!nYmxJLtA$nJMm`!WA6DP zy-k@&FV-fl&5ebdyl3R+$oqMx`n{W9dw^8`e zqwpF04*%V^>N4-mudXk^nz|=+S@dAIn_GtX>tFqx3jURVd`qZXRd^3>R%vAR?}r9t|8K?XQ~c3*EFR$A^(ajY ze#^JXOQLW5S`?mRjagrYo4MCc>(^lIpUBPBH{3+eUL`E|WV+_PiuG@;a$i^U{jXx* z8-DH?_8miSxY-!}6XEf6OSA|&3_hFcf`W)`5*1w zoy$$vJ?5lb_Qt!eb8XMUpuQ5q&U*Qo>zn-d;#vNY@7#ObJ8p=DEm!Hwls~@0pFJno zW2X*~9sJ+**?1`S&8y^%={FnK!(-Th{^)tv;Unf<9`MBzU;L79vGy7X}6 zHOgN4HRL3`Lcf8$3AbXfH5P6s?~L?>_ZN1rbJK9f`_r3*S>rMNniuo|kZp_fhfX%O zPe1BfCpR73b_08P> z@?LD;@w4zir297yME?gLz)>4ye31Mw_B_@bJ{s9`_i^%5_#pkm_$WSw&){?T0=|T= z;_LVZ9>+?oM&XV2>>$@;6F$SQ&tWS)%9FN}yYMHxg4gkq!L7b$|1B2huK$sLjg2BN z#-+F%W%`mnsjuK5*^Mgfx=Y+42lqc0+eP+Yu{PGX`Q=#OhLTwS=0A~D%+0a>@X>gG z;+fdxo-@FY!Nw292Rj;KgEy{^eT8|ns^8#^+hc1}ei|;JOL~c`Asa}%3%B7tkPRj7 z$3yrV$RZLS!(I9{XQD6hm}gxZ>%ZckVttog9_!n8X-w8{-r^;(!P_2)y+3mEk$Yo< zcikK7zwi2(s#s$1=1H-^vG0ie9sL^bHwmAh|2=M?--PA#Ral3Oki{f+;t6~nOYv#% zaVxjCLw1;uO(gE5zZV_!Z(;}j0bB4D?8i^>A9xXodDgaJ6;>m6nK>ER#^16o3Tu~M z6xPkR#!da?sT%E>QvDAoz(&X-6YusO|3YqYw&zyod~Q>I7a0_Odf57FatCwq=}}?l z)GPGEjtaY(OO(lb3Y7I3SA^mz`Y5mqJ5hq&xY9T5Aq$UP6gCfC6tnm?jWtX4kucY$>mxxaeV=@-%=KrCbCegVXRBAvMtuOUBouY78*vgTv* zhb7YeKTC^`ORF2@3HzkiO7GlZoaVT>BR@2U?q|l0pVMc^?&G!M_5tZ%n%$`HcW;z- z@qT?_7vZDc_ZQk}7faup?vdXxU(dXqEMs@&4C%vtPqBNQ``c%_&U^%A%*TB54_*6v zWYVYMW%pg{dZ7tTW&Hf6YyT#E>U>KBn$h>u2jYF7xH{hd?Co*qf5rQ*_>XwsgFlJ) zKlpomJKo0400tj>AwKx&|A-sLO}r49ANo~%aL1H*U*r0C-+kNT{ZBj-@83bcV^-+< zNOin_;!16z_lG}lI~2LU=W%uBS>ccLKSkz~PsInXTN_U_T@`XpTxI<4;;?$^#l}6X zfyNrF9lk28>opI)`0DV~*=xf3sp(C=`@-4wJD+=L*h+3Q zZd*h?&Agr5k$q_>CU-LL;?M2{#&xl0&ZS{5_O)FaO7YBzOG6p6qOM_s-3|2TpMG%mCLAI(dQ|DrX?`hOgqVEsQjhRlUT z7rOBmVKIy#nO=v!WbvK(C%A<9E_@JQ#7aDkUh(`taVfX&5XT=HEzaQq`g^CCKZR2K zD_U-nKjB~Tef$tp)#1L3ZPbop0 z(=EC_F4p#0e>C+fX)7)4 z;O8!KCraj|h27-dWogI;&yxpAH$<~7@x_i>P(T;z^b9fz>YUfPATx`Zb zq>JaD!e{U~{DHgM*x!fuV>0Gp5uU(TunphBcd;L*@Jk%QJEf!1I7`2Qe2M%ErqS=m z!}vRV8ec;;a-W4+gvn2h&e2Ik@0@`QhvC*-B*XTs$dxrUEGluP7d z7y7x=d{2z$_Ga9Rzr`XvinZ8@z>iCDEvDi={Nrf#GpLp%ULPgTO6A`@mxsH_sqF8- zUAWP;K4s94j2Zn(ne#33Z-<%Kqo&Sm3`yYIV{-4YBx!~s*ou&_jT{S-c4sHE3?OfbN{{*-n{3*VN>n;y3^26~wiE;e9 z0zaq!6gSdGx%N+_sS$}4ilF>0gr~^$%p2GjxW19x#JoA{flz2Kfi2`# z=56eYCOib& z$io`?TC7{)|Kln8dTem502}F>uz806kCF0!?nTbo(MDRR{3o|F@5oaAD|dI2yO?*g zFL8YjxtDoglk&e)`A?ShDF07e^U`}!h}KXYWUe@G?aQ$8pR8i8W?$p_VX~IFuF3i) z_m8YEpKE>7e0`$}&G#o;m|L^7v9is9Cfk`i*mt_#MRqe+`*$_v!m~zrk`>IAi-q@6 z;Z5%MughoqA9MW=avyVPivQ(bmysPM!n0g>y1$!jWp2w8-UY&&Z0z+v*f+c0Le??Y zANRj{{4a9DqM4xp8?g!NX3Y%wcna&0yJ%*}!y2qbB57vGf%%$YHFht&CzK3`Pv^uZ zatCwq?0eL=?@=eX$9$A~Leb)T!qZFd3ERod?kU6;Y{j?bLBE9QOo^AFD>^Yne)$BM zi;vUykw3(@@Ev*Ev&yE&l|?^h@an2o>1$MIP#K@s*uX770YrOHiwPV3)Sp(7XAKy&AIT$dVf~!ftJZ z66`szU%4kKJbQ<4)2{o1G4{R2(|?Ie`E`T#;p58JQT%Mc2O~GGeK@lB+kWR6TRdYE zH$QWImuo*@U#yH?$-JI<5`Ql9Zr|qj@0iDW))?+5xqbuxZs*s#wE_MW@8Q>D+^%)~ zC$9g=^{wuy!jJg#CvIyr%&R=(ySP(3_UpKn-<9K%!Uwrq%}t&D_|Q}}IYzG@(}O#EGF4bK@r>yb2&-=Y zKy$R7#&Is8g^zxR4h3;xdi?{iN=8zTp+rrxgn)BZW9{bOC=8mv`;T4$0^enu=j zwbc42>j*c{3#^&gXnn*ca&xw^K5`55R&raqF#z&u=I!JT_Y{*mnRk)9T`wW`Fz+Sz z4O^MSXit!PPMFIvU`_{iVJGsA zTbqxySc9B#mq@JFh1KJ&>$g_FfZYac!bao|m^Y2}cnVFO=Io$F+G<99=XIe@KUPDH zIX&ncHfIOju63b}-ir2f=JcT4T}iAUY8zknOLxq;n7 zWP`XA=f@g+AD?nP?*3QsVfTDaT%5_TKVYKgRJ&)jxcq@AuDiBc+WZ*?@dwwc-2bxs zo)=%g!~Jvk8vC!iryF1OyshqiFaLL;pWCl{my7we7Ju*#S9?~>yVr96ifg0!@fhyo zPYeHl?fzT&b2+yO>vnU7ZVsy!YyWF2=aP9#wg1VrCEEYx+W$4$|EF)!|Em3u0&K)4 zZ0^(k#};hGHWcA$Y{!mL;~&_GUD%Bh?7?2_TV(u$|Ic`4nP=|z4&~$l=7VI#LT&VN z|7zCQP^I7Uf88JD?fp&s%l~q|8oSmr;tYs;aU0b%kv zd|kM1#g~NV3wQjPvY)Z z(SIf1;lJD>{O`t<+rwo0|F7Ni58Qv0f4?DrL~hr3ds<(55xGsj z_||-Li*n2{IyW&4vd;C}5?bBhqoG0JVaF|5{*T$O5$ zQIg66x#^s~dK6#-)~A`9glLY^I{^T@bCY zK8iN9;^34~;Q-1tO828IWmI?urSyFm+3$FQ`Qqcu7bmN{*CDd9?Z!~y`qzcgCgJm4 zVbdV&e;NH};di^R`GK(ef&VgyN!%S|_a)(Yv2efBJ%1s8k3S3ltAzh6+|8h0%-!#q z2f1zF?-l;pKXNxyT-%6Wxc1+fZ{p_*_^|uFgs);X;@;t9a=T}IiXWHq<6eB6{ipC5 zd>ePN`w{Nu*C+86+~U99jpew-v)+%t#Z!11E4<%k@6(EJAu&t($11EwF7mJjYq9Rc zts#H5^ndDB^?&Jqf%K1`yEoT&Y@VjijogwM55FNlBkYQ%O8--&|6b{TK>9x?{gb~MmawZpr8J6;qzW$aVW+9pxjoIHEG5_$=YoF ze`G!L5wgMcMzV>ynQY0_|3|hlA0^ve?--bBU8AuTbmLR}{y08^&*5>bL>XG}6FeXu zd;p)qYWxJF{F|@hO8@6}d=6hg6>2fcf9c0f{?h|kC%o=@(f@XDi#+3B{Hs;b|H41v zX}rrn`)_!s|MV{WH|)V#oQGxHiOIN(UsvH_d=WcPhka)&X^Q-SR=p7p87k<66W3HUi<9tLE(&>!e(|`t;yelBJ1+EVSCO5ZTv~j zBe^!LJ8cc~xogc4zBUxhn-n%IyeVuVH!heIau!}|py%4KdWyB+8P|rqG<)gO*Pg#N ztg%ME&D#Ig1?i!kJc>?h^g9O9jZIIohlREKXPhx`%G!PJaD;4P*XUXcb2It>(Dv@( zQC;=^|C?PYCnaMqwYf_q|NoWbAl+u7G(PG6GEh=9{h5lY^(&O{|p6C4gduCqiv-a9+uf0BN z?aR7vx9bMAl<{o>q{q$L0m!z%4SLt;$F<{Xyg{{44|UK84bc4P5;Y-h-s@%WC2kSo zh)ibNfj&<3N{ zc>jS1dn>e+as3mHuZZiPV>%9eMxB&jyNF*mVo%{ji>E|GRJuKF9U` z1-KjTfrsIFcn?0uJrF#7MfW40he7yH^X!5GurZdyKDk7R$5|iYJm1h_+=m04=NNBt za0%@%Tw{|``uGxM;3xC=5@j(CBzr6MX`1aWOjl#94F#k?E(W}yU1FEE}7 zUVD#v6Z{>1O1D}FNfTm8{RdDP-X=8w2!&#Vu-@sci z_cPRs;XUlXhd;svn0n!U>mSI!!DaX>T!lZXwYF2r z+Qayt1GDsL?9Tm+|J}>@-v{aA$4@#k!^`;JUdDn(82^jRVLSIYV+6R)@}U5G;W%SK z4|DyEaQz+R`n$~acZsnDxL4p_iN7lRMaQX4asM-hz_paVc`$2@G;E|!vzfX~0%e<6 z>M9GUv&^OZyO=u563V}`=;NP5T?TjaeW2*~h4)crg6{p4q4rRg+Dlz(H+8CAl#w2# zthAFd(<;hNF_fVQ^WW<~ajtdfx)wU7@3%V7-fw*w`<<``w!qim8Pf6u`SOK@j1hp( zz_qXh>H^eTBh+J=59I2l-s+{^>ZTs+y5DMSyx;Qd;kmV!=U5T-*gVF!ohIH>=&_II z*na8_d#E?>r5+7=*o&#pmK>lSy^p#nPudTlax2X5(`8M8|rI5n1eObOK67g29tNIiZr^?2&S^`vPXY3o`@ zUM!kPbCdRsq)o$S(%huI$pdJ?-cI_PZ?kyu+r}~72T1#uD6c{X_I{pKW^Hw|#$pfY zA38g;Hcj9>}!QK>|XF~ zBrl+28)*++JIM>^!QKmI9qfMi|9l^V{bBY+U|=J43ph)f{pY8to5ONg3%A4F@C|ql z%AgTO;1C>vxz}GFd>@{MRLFuND2Ezo05|yHAiN5{fq%h&{0C(`_!8U(_rhMt1rPK=2!`MtxB|9u z%Kwl!Mfo3+u{*$diSj?BVo!r~_GLgO_AL1SJ^xuOr};AZzk+oXH?judVe)^3{GY?x zgHz=HtOeYQ^sRT2|IkI8qVvCwQ~o(^#zv9ru1B##q+|Uj0ai4ww|9~gC z#-4?r!YMcd3pk$Tlx#P{2DqENxQg&T0iTBwsDvK)9btS3zu>q}> zU^DLD!o3LS?z4TI>)~3s0g|}ZQehNchrhtz;k)=tWPnkU8Dzu&D?8FT!gmr6&r$N! z966BDad(t6QeD*9*_WL#GtVeK!2Prt9H%W|7vt`x=Be-!Z3_Ds zcXyHb&xdI%V9Z1r<&J9h)f}9sN@Nw(ZDVW&)UKe;j%>0sPb!Uh@MUwD2TENXdn>en z8+=(GtuKT0I?TLx$~Cx0*Qa5-4|_lKKraOG7ozjPk3GQt0k#MCGY=jaVcX`IEqm5% zo`18IR5V-3f!T6^(>+@$kXkldY16Zn4jGUMSy^Um-)zeNvnl`2R-Sh@^M7Wmpm#R& ze`e2&e_?E2$qxFz_R;^<_`mdj(Pve2?rPOCro|P@`UflM{~{g@jNxx2PE8!!JjHyn zz4U+4cjciEt93E+&FE0^?qw*J&AaLl9ntKfnd}JG@AEo#26`9P>$71oy&LxE}_A&+yn@LhgfdaKT#K?t-Ub zKm0dTK^FwzSMWZZh1uw??0YXE-&vtWPOD9AF`3{CS>zzo`3shN87ADyt7T4bsKoIc>Xzf|6S(&w{W&P zxhA@n%$8YWvxn=!e0M7Ped)72{|F;MSV6AYkc0fk9&ydmbLf5m(qIRC7mA<~9)T@9 zQ@#r4uzvtY;4N5;+oz!k9w#r0R$ZY^b44W^uc#QAx8RC$p@8lDSy$+ryh5AL70Mk~ zXh*!FvN=~&ihBjJ92ys2q0i$AbKS3~Y2g*Q*jI~NJz><%y+T{h6|VU!jAy+p}JsMi2CJTpzM?&lSE2bVc2}uc!+mo3AKLTm}he00K*|&?j<5A!Lwq z+Rl9_J0xEEP)U${_Cq<&e<%G$KQf2yX40S* z@*k!A^C0D)?UaA^Q~r60@(=QVE&t%BdMo))J+YQK4F7BW-%k5Cw168t&0L~9mt^jlX?L8 zbBN)ddli!5OWaczxp!{nI>;dZe*s=-XPY(DZGS_)jGrL(G3@p5N9^73ZNjRCAK>E->2Dt8 zo!g6av+afM|3kdscQix)d@k=fxDEG4SPSdnb712hwnKFNw>{kd$1lr~c3IBd-2YXx zbOU~t!5830xCMSin7;vv`Zne_+Ukf4(|T-Ya685}#eW-*ciREvW#Ug2-j)J+V4-IP z@{`C$bfLZYxciLlzXlB&;D`--Ud54elh5UWU>l z-f#9!#gj+>1rzK$1}|cN8<{)xG37x%&!z$>bmv<|Bu+7zRpMohfBYOl?7iC&@KdF>=78C{4sdI&-$O8>fFrx zF@g8v0-le&1ADi2F+Q|Q{fkjN5RjdBZXzT>GW>~fonIi&2qzUkX_q>bez8Lt7dq&} z=wSXYeHv4ggDCsta-KZMXMYlLElex0is%<9E}|R+r2#Ylpj+jqyH$}!A4(a0DNv2Q zW((tA4)Xqox~1K!A7T7U9?ySh+R~*b2(RTl@BfRu|1b0YM}7g_Z-ivhChzr z=prq1+3pE22Lahf`kFPC{Y9h?euBu*Ueaej>GKllgB)Z#F|XcA0$Ua3zp8r68K}2X zvg)m5aP-z&=|%Nc2K!Sx>#el1dMlgybdHyKQ0&j5PMuFVu>f)>=TQD*o;rigOD-~R zePoV`7+777H0%47j?ba|$2@oZ)|_VEJDBzT>SE@oe$^b+E@fUkG(qzU%6|!_{KvXS zizxpsq5QX)@*nsXQ2yJ(I!Lo9|3TMA);-!x`42idrjKL#39pB6dLej$e(#I)gA-2R zGX36X>Gz(dAN(Bs;E=!MxC&MrS2}6-JDwNCOOLAr3O65DQNnT7Z8@$An0lwyI>+_+ z`pXX1-f&oz`*8bZt(C|mgCy?>{gGq;FF4&NC^w!^GB_Z8*KuXQRC29#p5wEz=RhXM zXTb&BX8nQts`f2?4B5AUOA>wo6Z5&Akl&^~vb^Ul#t=|1fD^c%gh+3S6&*LQu)wO_ zNBavjon~Fr&+*)(?W2(gk%tON>)Cl~;XUJ?^$Gd*(KgAkZM`$!S_Fmre zQ>>YPe7@SCzHGkgs^+t1!hF?YZ_Z+DN!onM$n(_zA;Jyf9w_2l49LN$RO=j3Z6ck* zoU1!9pS2<8Q%9Vyj&aHfjq|A+&R4H{KJ#eMYkZ#miT|Vo)<9!Cv}5blN`{m?##gi6 zSX$8_Z6Jfnun#JowPLakT+OvVPdU8{lx+_ykB?>Nc7}PjRayCYRaTKVOdTMslH-Fa zhO&`Cl|n^ljaBZZ&f^|X<$lIfFu$St0OQMN4XTzk$m)waRp*GP(H^1h6XD$-QIorq zI!{C`xVPfw!S1~fQCn6-?dKx$U5u!6>7Y8QBI@2Y&NCpQURQ+i6cP0!`?hr|uxFe( zfSn3)j^N==MaHkD58!GIE}o}>y(H4*0ojjNTZy{|lr%D+6fDtMK@Mitt}NJ;38zEvI=;Q7b;Z;bhR{0v6n#Ur8j8*eS`MjH&ik5hAO8| zsOsFzSelw~^q-vK`F~1vP=ETA8b(g3@$e}%A)7CsqW$9(;}K7BkDQ|YPx0yTpEA{gcWywtkaxj0sD~=}F30kTRof$Q6=f@PNoE7LpN-F~ z*gEI&PJ@29pD-SQc|0dS1&g3_jPWP4Eq(A+#z4JK`&FT(&*I*^fHe}o64nP_po?JMtUH4a}^(h|ymWvuy@#r+Q{+#`9*srzA1wX@!H z6>Gsk)>(7^U#*<;w1sdlWSC zs0P>ZS*(pq-4E)Zad(#*E>f?!Fw+L=o}6g|ZAF?kP;cx^8)!Syw1Ez+VvTd!GrK9< zbwMv>y&mX)i8fF$Wj{YGNADo}SsPbnKl!Crv0d@XD+FANQw3Elu z&vhLzfHQ+Mx*5Nrh0+I~q5b?;-v7V1v=+B4)~?U*Asv50`X|$dPns70mHcS2w1NGb z;R7ZV-i^E+?u9)3eb7dpFsbk%v!DFiVcZ|L^ljw#jGf7(4YLUcng1QaXa4+d&=3EYtzR6NXi3-LcrfA(Jb9PV7A zcJ>v&Oq`J2WMc8(EsbD*1OC9h{%hp>a0#yCUaFx#_yz1AyuU=1Mz$xZOoymN@NoWy=6>tsMxIg2NZ(U6ppJ&r=xc|?> z2dPW+9`cWH0sadAfXkpy^4x%r0j~wyeE1A}8m@<0#tXP+(f1Gaj8|xwbAuQfXj^Xf zExBIB_GXnVXw+vZhkYG4mW8q5H3` zv8ai1Ym>5GYEt%|CgtpIQm(5(c`KS`-har#rA;b2=%FpBiF$xXrRRA6ZS|-;;It|T zr!vr_s*6pkzR*NFaFc2mHpw+N&8pkhMEhsD)xh~0d(*9^jZL%zd(^ViX}O7?=TWa( z8`G_}1u2#{wn;wxwa;l{Y)HD*c`)7TLUu1{QqK<3@pO~=Mm+SzHPQaxq`;(ydO(vx zhnp0hYElIEf$=5{jx^~A={n6aznOz;XXLLA=P@l z0sI-P51wH?fZHR+Jr%dgREyWK?R#c>cAEA2aGLcv_8$Vv5!s%F7vO_4j6NNw9)@#vyyhz((Yd? zM^UYud9_M$)he~KR%z}WEB#z8?O&Nzrn^>Ii!!b3vRdV=YGZDp&B{ApOZ#^#^M5m~ z!t-{kXtGwt_%AtIOZ#`GRkkbBDo0jCY8g*oCtf+W>fLRs*;mVYakL>o9p|e=-m#o^ z5zg7fc_vA}*JuxL6TatCE%o18we6_WHMsdUzNz-(we@*w*1*PE4PMT)Y-KL=&$1HTE+sAYa+6Z_iMZrE z?@`L4EGxCjW~I4YO2=PDkxQ9Nv#9rGS=q>(^DgBctXJN4m+~LUNs?`6TE6! zbzE+TSDpiQtM#BuZ3kTPCb;Ayoc38RzAxjV|1;a_n(I>c;Vi3%^YvcLvik7XKkk)( zvE2#~zu*!NebHG~c)QJt>~m=Ve}h|z&!t4G%3G)E%Wl=o$+k+<>X@&UO?kbJ`AR+& zcGhW(G$}q;r;?a#EB8_zb)Guq<6dB|qd)L1)(V4Il?s_~hS5eqF3na53d|WY2UR-x;V= zALs37yQ8d5oq2WYBA(rjI=)v_N1e#WeDFH8=y$49{Z`_$Fvqgb%CQoWNvG?n|G4B>Qm=2IbIQVc#tqafZDG68k>AAa zQOLqQdu~1B|LeJy$^QlQ%3oZsf<^VDmrK)Jx5ZN@RFY7yQv8-(XjeJ1Vx*n)KA{CG zxc=QqRt;{o+sJ>ysUw{F*m^Y}8y9CN-%b?)~+wos(nrPUcvB$bRDG-{n%^e7(-BdWw=q7dO|3<3YP--TFX8(*U;EiM z+V_04e{uhBdt2Se9#@-sx74Z+*}tFrf6vlW&Iy7>mPVkv(-!cpI5HK zUdBIosRMY`u&-T>7rFlzPpKK%GU=5&)+Gf4ddre4$G&h395j5#l$%J9Y4)-73xVF_zoTI@LZ^ zEJ(B}(|puX60PchPc?ISR=ZECD~9%|OWCY*&GX=rPer{x6?gg=1M1Vn&9tR&V0`iT zNz%v1^U7y?x+j4F9vA#0pqO*xs!}c7jai333#H)F$kC#EVWw*C0k$Wqtx1I7& zlI2*KWI2&3huf9v@+vKc`=5I`V`)43wJU2D_dhacNjvM`w<~W~yYdgTt6+aSV{yoT z+>7V7GykVurFQZkS$^&meO}!Ev)Wa~y;qH_InMo0ecjc_{l7EEs-JFG!%p&lQ9FHM z?P^}yu9nz#xmVd3N7b%YZ<5uvs-65#vV4o%S)-s{9UF73&b{sG+T6f=!X&F_Z?@ID zqn-6H+SR|io$^nT6_`%4g2>S3cGB0Y$Yt*Tv-PZh(N0H#hw*PMw0-?jNyy|~Eppf! zTO~D=N9GITWI%fQQmlpm7mt4g6+vx;dl%E zU&&T6{z|qru>N(bRd%>Vvm>o&^F{cCCe-b~vdZk}DOT>rEQE}?A@>BCQZf8xRN*e123|CI*DYKtLN~b)yue;z9PTX zkMwhVUoJTb=POJ2iM`gW~=W(yFdz8G#BL{w+2PiM?_t0_7^|Xun1TyI*k21MNvgUYL zFa1}%FFm|ZJJ4Jo4bLbt}gaZZk9-&#)TLXIOP-Ic_?`stvg10^j8ft9>NHYCG+bcPhj3@|X;(Vo`=wx`2F~n_;DKs*mw}uRuPs zhITV#^-c6KAZezwoq-0P)4xOh8Ta>*ESF;Y6Y_?2^gr!nJ==w`Tz|3ZrVXNJL9BXT zVhrR~*6?H7&;Gz!#z4-ERS3ekN0!8DU{Nf6^0B%|I0d{vK1G<_|nt1Bj*w=9!pB2?T_)Vc+aSuEQdms;< zhbHd(=2>y9JH1AYff3bpuA$F?Ho0AES;pI11^n@3p!bd#&!OUdy>+4fPEAomZ`q zV-{^u=hi3*Z0Fa=j(aw4ISbd!j5}DeM(JS2A*4duqBYdBXw#!zun<|4w?_GnH7dwj zqui-D<&Dtxgj)^mNR_8)dt_hvKH4IS*3chE+a&u>5y!y7m3s6&%7?2~>UDp_I{ckM zYbAO8CFq#Dl76+7%&%Fg+u80pyp;I`OPOD=RQ*{?^)=kwOIA`pS*ccN+p|($@VS;U zKWC-B&VJYBrL=D^)i<#>Y+R{~)r{%ke9ec&E zs~X5nTSx7z3ZinuUN|Sbt!XCmeOvsl=9h1#;302{Y1Esd2Xe$AbaUjPHw3PP!rFxDyI07r>y#J2GFOf}=gIkUoPJE$`S5*o%R$GVV}A;}57`IT zKpm zc>g)Af`G#+++nkdcwZEc#H-|Tyh_~#DmxrcKXAM%PRFYfS+&<`RUf3y5Nfx?%LR2% z4-L?`vp`MAW{1;iiO~MDFJ7Lr)PIp}TjS;39xq>BCH0>Kb?n@r&Q%Vpi+igZdT7V! zol2m-NI!6l!}8C`<@w{Zf|n8$n$=;2&nECbN#K1loB3a)-z+=dze=$Z7sV@yaFgj{ zc1$O8UluE6XFTt#IZE3bkN$Qm<8-o>8Jl8d9Za^eH`=WnWbWw%ASJzoXL!pjNt zeajPRwpoqa64bOe zLCw1psQ*;S?M46CjcVPJY_*jY%L~3vyVc$qua4>2+@Hx-S4+ zVQj{^6sveqqE&Vr9abb;70B`!+6wY$?{%+J4YK-Tp6ncF+n#781xQ22fjV|1Tgm8^ z(%WS@*_U=;y;AqBr|l%gN@u)5)&bIRKWzmY*DHJ1dUZa!P95V(R`$5@_2TyllL5H$aXvU$mc``3}tg46hpbWAGlvr0?Spwd&I|fA7m_C zCI{|m`Ie*8NYIPp2RM+ub zbswyx{i9O7)4BA2Rx-D^QvUHu1+WLFDyd&qD%@E~zeS}6+?5(^tYj>}YW4T7R?o%N z>h-Rs9u=c*_iA;-#xSomM$d5$CnQ2S1Y=?pS{Nh0eKq5bVpykewVKbZRyx;5l#@tU)gH4Z;hc|8Y3?Rac|ucqn2GUa<7OXomSIMv|5GxSCek5X-AEr z{I*)ToiWPGiqR|he+w=_4(BNM#;77MhB1_@RpyS7lVel%tX8r;hI(6!5)Z6a(k>qM z$75vQx?0(bSJO`)qs++|WkDffWE_l9>U0e4I$Vs?mDK;LssH_f`d>BmziPtCmlK)N zn@|1k57hti>HGXLczwu^<1U{db-^rhd!%AM@0;V4k{>Jxk^>CU&0s zPX9~&OMWju;RFaThzt>a2pzshyq|>^O#HWBqr3-Mzw=Sn??e`|T{MgO>jBp9gwk!S z-wEYASikeZAyu-!>frmTX6=L;FdhGPNZj^GC2eIb5TqmTpq$uK_OaFnoo#7zSnI=e zP#LpW3j}-C0_rur2bGh>_gm8DsS~;!#NST*p*M^8v)#`+pnmoRs)+x2;(nIz_f5}Z z{03uJrucr}ang@CyLPgED0TJvRjePnk?%+D9Af>fgRGx*P%Zn0WY&1}EMbkuMTgWj zmo*+^4$3!&@gtY!aUt>Dx0e|EL!W+Df<2= zUuOQx)yyMf{cpyi<<5FpdHeYO9~3|#;{uBI&u08DHi&a#HVW!5r4HnH7|Y+;O{o4FAl=0dctVt(jG=7)k0 zd;5d0spCQBchV@`h3w}2+q0kfol}2QANKy;%n!ZzN9O-AKNSBV2y<)%2Cxr;ZE804 z%-PgCW-AGjDHl1wi9H2U*_Q_CjLFMjj9n%&D~7QTjjt(ZDf%xsPx~+W??HbkJb?Z$ zF%ED)eZG4c_xIo&l|9NhKx9P~f`&_^Z~*cybf=|@8NxzP5pB|TnB65 zD97G}{5))cufPLv9QW@a?}M+xDCc<``4qeWKZMtC%S4{UuOImjcneP8ehiL49d&~y zxPtrdpqppYrW+|g#4&Ccu7|&H{3-l?3H#0PFkHs%G34#=Bs>GpLqRX|`XP@#|6C|J z&bR_7!d?g!=UEp3$`1ZorCG;SgS{H6pc0Zc-@y1l=J|sSQcm9>CpaJ(GA`Yq^b5@4 zMyBqd4QTVPl+8T;tSRQwGZ((Qk#Q@~Swvn!KXViM0_0)ulzMu}vuW~<{0@*;e&itA z1DD9d3*;rTo^=4~ihdEo=y~sEk zc#$+n0*hyQ2J*M0TM6>xY*!%PWxE;K4MQ*=_j=NDA@a9u{|E9<$iKn6a24goYvFV7 z1-Q}J@pBFG4!9fMBg}`9pT~U@@~-e<7?pkbgsd8~I&$5^N7re>}iApnX%c&r^TgPW^E| z^+)E(r65z;PQyK&xpEm(rlO`48jV|IjnVwF77wp-|y9(L2hj$racd*@!?Ape=ig#NW2EF7N1aSAW zKeU~9S{7p^Ij({2W@Hn`yVzgLc0IC=Q_$|lX zh`bH%fLq{h_%>{UZ@~TVF#I0Be}?mLH(|es+s_~gK6ajG0`hO{yN7+>fPZ11h5P;3 zo7vY6-7p9t7=C7Zebcm7&sv$+4E0xF>js-b4flxiV!IjHLKtqOhwWBm8~eRTAKUH7j+dDKgzRFw8`(39@xR2ik2v>( z9~}Y^#2)evE4+yLPlPdm9ORhDan^)~!F?kd*gv8Gc0Yt52)#wDDG&Y72c6IX-OvSQ zE&4WS2Oqeh1zN!ajf*IgLNhdh3u>Vr>Y!@s8p?!}rH_xO8d=76DY9amGB>iQj52v1 zW$|;YsgKM%H=^9>5f!9SX7Bw_S!bD(4>^zxX(P<~uzm2>f`3u}0Lp}p9nr%qJ?R)IE@vg8# z;yL>NAo&vWKf#GT1yb3U2I-WwGah99neC$gld-0Zeau1T9%lWSgip7PoR_1%4{oS4E+-oj8Obg?MDZAoktFCkOkFkk7yiP{8(w?7NCF zY4hM(_&M%3a}VDFYk=Xpii5Qf4+*di*24za2w#Fta4T$vFT-u{GMF+j-^|kx3!yn;K@Bv(aKf_<(uW%7e`}yC%jQu4m#5c(G zPxu#HhAZ$P*uV~nkOax#04JnCDx^U=Fwb9^kOkS01G$g~`A`6bPz1$L0;Ny}f^O)6Ug(2<@IwHC5P~p7U;w_& zwWZJN^RL?>36jA9PDp`N!|V9LG=5SKCt{{U1{4^gdkdm_3!-}qaV?Cx7DinQqpr=k zwm`Q&Z?+F}mMOxR$}~*nVP+Ykwrp(KQCkkSoTx1qTW-{5{N_z{VCL)dzfCMNM!A-1 zDgWE_f3!@?wL&ZP1$|LpWR?Ff#;!3&)D~xKYmE_aj09t>Gsb#jY%s<~4C5`%9A&&3 zpT?u{XS^9-#?uzK9qxcTVJqAPcLN_4_~JJB8r%c-!glyNd;{)-9l(bNzIZ=801v`L z@Gv|AJK=51xbP;RV&77#28#VKyWV`iDXXS0m|v$qoAbWcVdD9IVw*9Gl8sr~ z@P6$_mh%-+OSMU>_v_73?{}COz8^4gJD+4iKA&mQ=X??2uhMGWsGAg{n@vB!&2S43 z{WUjf&COc#8+-Cvt$8;wz&jxJCdJ;Y*fso(ASH}hnTi?pfDf}jHo7A=x+6BaBTljO zAjHN+cf>_^#6@?kRqR@Jtc~ti8+D3TY&=f!QK$H*Q-Wd>a7u_eB}ARpDVDzX*mY5- zby27Fid~P>`l!?TsM7|;Zop|n)M-Q1X`^B{;RB2-p8BETP z$tf{83nl?$5>_;rs6r;FIK|y$`by%~@HfKjj*BKvT&&`vzT=`v5{K7la>PXwB5tkX zqKOa}O@z33#YGb!E}8&w35ts6q|-2@qp zX}scZ68S~rHDe~?;}pME@i->bBeT6>ZU$2esulsYQEr zyKdj7J9NjLy5pIfbq8)wKA}5x=ht-Sw{+)Eb>~O~2DDXo>8>r0=x*J8yYBw#Pj&Zw zyL9&feN|t*PhWjVUwuM2ZqqjNhp%bdgWC3xwmqV+=^ov4yY9iE2iRwO!k9*Y-QK{Z4HsfbE33{hQkUkhVXp?GtW&U0?s2zWyV9{TY2j_vyZ`>c0E7 z!$Z37o4W6Tt>Sf7f$* zOpo324D8oqyPncxY(B2X_VSN|wR&8S->%1Z>G7xZ_+I{Tfa_27IElGiyC2f-=bq3H z^aJzX15e2a%)NR-PY~n}^+R;~;V$jb9`lDEY0uNz^JDG#iGHLf_2d>kiGEMw?a7Dq zM1}l&5_2#AI9RJEU(l1Jdm{k)#h zGY{!mJ!=x~*)4kZEv2OgtdC@zp4aow>G>D*g7)jj`tctA zQTFIX{X{?6!+(zs=%@Ob{#!5Ul{&pruU8uNN|Rn`(<>c!z+FP8A)AKm8cNhql7^Bs zs7*s&4f!owe<;YJNN zX}DR#EgE)f*rVZA4Yz67t6`sp+cn&w;Z6vaHBzIIT8+5Gh~trZjWlSaQH*RIacjh*kyeegX~e4$pGMj>(xH(~jdW?GTO&Oh z>D5S|M*214*GNDkL5+ko5*CjtZmw6e^=htOt<|e8z1pf*+x2RPMr|6kYcx@#SsKmO zXpTm6HJYc5<2 z(H4!mHR{o5t47;2>eZ-EqwN~yTs<1?)u>;i0gVPV8q#Q3qY;e`Xmn6xHjUXemZ-5L zjU{W$p)se%QZ$yTu{4dPYb--!nHtN|ShmJ;G?uHeJdNdRtUzOh8Y|LRvBpX?R;sZw zjg@PxLSxk$tI-&9PMS2(E%I#=11tt+9TM=W0Ap z6jW>!hqT|hCH1fDx z;~tH-YP?P3UXA-S-mdWujdyCiOXJ-d@6mX##``qR8T}d$XgsL#kj5h#AJ8G24%u}m zQHPRrC|QS6btp}T(sd|9hcb02ONX*`C`X5Kbtq4V@^z?4he~y*Oou9Ts8WZjbf`v$ zTsl;zL-jh;phJy1)TBerI@F>=ZXNRIP^%6FbSV6^Udz>Md7_p@6)RddqcS$()I_Q# z(ln8-iA+soYa&Mzxthq+M7}2QP^gI_O%!XQL=&Z&DAPo_CMq;hsfj90RBNI}6SbOf zX`)^e4Vq}wM3W|(HPIrTx)UBvv}&SF6JAX)@V{LX9h&IWM3*MIHPNGqUQP6AqF)n! zO$0O%)I>-VVg6g-E_etYh9}@D*bC2VBBF@_O$_RYO-JlHlBgp|I+Cm-4jpmoNQ#c6 z>PVW7r0YnAj%4acmX2iWNTrTc>8MRd?K+yIqbWL?siRpsnysT{I@+P5ojTg3qun~% zqoch#+NY!aI_lTafQ|-rG^C?p9gXPdfQ}Ao(xyqfCKEN8q{(DWIyC9jWQrzJHJPr- z3{7TgGE0-$n#|Q?o+b-4S)|EgO_pe~R18X)tkPt)CTld=r^$Z3k*7CW^+vni=+rU0 zjwR|?j*jK(Se}mM>sWz~73&zoM9XxnT*oSOtWw9SbgWv(YILkl$Le*gLB|?(tVze3 zbX=W*+I6f$#|XSj$GUZ_N5^_~%&%hs9dCRqQOA3ZHyWeW7+z!a z>G%M~iCq3~^&7))j4;NDN}b5#Z#MGY8q|q={$`_qzmbLfjVyXAU<_=<#wayL*;_rv zz*cUI3jR@P!l~qMHmY=@+Q?dCxQtP!6ZJ+m%ozA>GPY)8wD335{gH9P9<`rHL`3bz z=HYMlccIJMF1=l+x9jzGgWhh`+fDr6ZZ<}nF}%j`8Kct}T^LgiV>peGVvJN{q!}X} zMr+#hektgwYaE7R7-;C$z z`8+0`9&KdKkh4f-nb8Iig1(Ozh-|53FtqCRI%&(@4jbCj_WNK{9R=%zU%+L};} zFg|AN*wDgfPVF;CMUx?#2+<^H*V`TZX)iLd?|i#^Ds?IuOVr_1riqe_O5OZT7#JoL zgbBWT#vYAHw>b~SjC(ZjZWCs=i5f=q=ozKhnsG$g?x=qgVJ?s!;|U|WX~OO?+RO;! z!Bf2>gS+-}kqLIUHjx&NI12cAPLH_30 zpz(|GKLWpI%lI&UgV?5Q+`AarQDZ94h@7Y~W6w2q)9i?d8u^;C^Cx%=uQ7a4gL6#T zKQc_XcFur-Es?+RnK+e9>LT!oG2>+7X11ap%^``#mc;+-;EP^YCSzpwJItBPO<)Y;(+I*q%5&N{&FsKKx;YGE z#!lSRXVO01#3|jxDc!^=-2{vAKf)Yo;>6u+3<6A#hG0VHKvI$;1DXnIDx|3aO%3X# zO(&CdGFc~`I+>!AsXCddlUX{Mt&@2=S*nvRoovv_CY^Naq*o`~b+S_@yLB?ClOdfP z&?%cv*>x&WrySz#$h+}W3Mb_Ccq&b&GIT0ayd}B9PZj7)L(!RrFrs)yl+1|A8R41{ zbu+?^x7HbJ`;6$srAKF)TxXgHqp5C2v`0mAhR!r+>P$=VnHF}ol+1|A8PPf;x@Lqw zD%xCU+VI&{7Zu)Ao$(TemoR*VI@6vGY`3%B$-ZtIq=2!rubX|n$X+Ay(_apiPzBXG z6UaK_@6wrI>zQELjBx2pDC0C7(t-YKL-N)2#a3dgmPzHRtzBVF6@--iD-&e;}%h^D>h?C z(^CwD)kC`W5LOR5_u{vYIQJ1>KmPsr_v6>kF@E&%v(L|Y{KU)8x%}j#pY!?2M?dHE zlaGGR>n9)m zwczguKLj8M=!>Es`PK}`qHKY;!L^beqa0R02#A3*;A`UlWIfc^pW51@Yl{R8M9 zK>q;x2hcx&{sHt4pnm}U1Lz+>{{Z?2&_96w0rU@`e*pah=pR7;0Qv{eKY;!L^beqa z0R02#A3*;A`UlWIfc^pW51@Yl{R8M9K>q;x2hcx&{sHt4pnm}U1Lz+>{{Z?2(4VDA zrvvC8K>q;x2hg8o^ri#oA4LBk`UlZJi2gzJ52Ak%{e$QqME@ZA2hl%>{z3E)qJI$m zgXkYb{~-DY(Lad(LG%xze-QnH=pRIX9%|D;^bDeB5IuwF8AQ(@dIr%mh@L_845DWc zJ%i{OM9(042GKK!o2U=^%Or(KCpiLG%owXAnJu=ov)MAbJMTGl-r+^bDeB z2t7mS8A8txdWO(5gq|Vv454QTJwxajLeCI-hR`#Fo+0!Mp=Ss^L+BYo&k%Zs&@+Ud zA@mHPX9zt*=ouoPL*#Rad=8P%A@VsyK8Mghg#ID)521eu{X^&h0!aFUSaeKqgNQc!sr!7 zuP}Or(JO59B45MgYnXfuldoa&HB7!n&^Lm<5%i6qZv=fK=o>-L2zo})GlHHG^o*cq z1pOlD7eT)W`bE$$f_@S5F+x5@&@qC35p;`?ZxQs1pkD<2BIp-EzXTybAnpUieSo+R5cdJ{ zYJj{NK!*Wz7(j;sbQnN~0dyE3uLj7g0d)C)sk#sBHnJ^EqnKk)C`n2*0KNC#ThXTz zIf0r)r)ckg2GJW*zN)UZ-d#zNU?6tT0TJiJdt!NFd185Dd185Dd186;HGATD@-=&6 zdh#`U;(GEu`Na0b_QdwY_QdwY_QdwY_QdwY_QdwY_QdwY_QdwY_QdwY_T+2##P{TD z_r&5M?d!9BeYT&^_Vd|(KHJY{`}u4?pY7+feSEe*&-Ulp{yf{CXZ!PPf1d5j^WU9+ zew()DtaiVsb=mo6 zWWt)XraJ$O4q9KV5o^pEw|w5H&l}yh4y`l)`}aTp>#bk^xcq0-Mo0Vq`T5q^KI{zHn&f#$9pRFSI ztgh9w#;pl!(wefStr=_9n(wdw^W(RL{(lX5{O7Gd$NzonwYTw~w=Vzl(YLX=f8Y1E zu=Jm|*3bWa%f`#!YA19(61f#kBc8e{hRv8d4K(UNB?fUWtaNYi~jmCbnxFd^pAhvcVYK` zZm)ko{dfNu+WC3Eqrb4f-esTvy}wI0yZm)emp}Uc{{4E(e*E_9A6MSt_VyRfVd>wI zf7=kvgn#r8*Zy`Zfxc4)eWwgQ{yY-@e(8Sqj{bf5)*1Y(zkVLK{+IICTch6U zEBoI|Pu6pXwZHzem;EpM(4aMBeX+h;!`4WD{k+HiZvQJB@}(ad`+0QxgrV+_p-KPg zS>u)?^z+}7KmI-BLsQ-}_482nw>{loKS92}Xa3=5_}4e^|LpF+-_ZS=``H?r>#tuo zp7-Yb58vj#6Z`wq-++Jjzh1xIa+HVsoc{a~_|NxQe~0}@?ESpgK4jtNNBTEi?5`j8 z)FB-_`n8oM@AG3kwA|nB;PcPV>TiA7hkqR(KZZjqKaXDjSo(qN|ByMn{?q@K>hH6+ ze0h9X`fvGW@I@M0>#v{RCH?>Sy7#yG>#hF2toPT?`{?iEMt}W0YQ9^3zT@YYwQunw zF|^tLP~Yv_^VdIqZf)~N|ETwmxVL-@U;TQ^uK1SW>)+2-fBpL0f8XJ|>F4|UKZn1* zHvK*G4L9VMjGw3e59ZJR{P*|3|Nk>K$S3&-=dfUq3@b{jW&>>*L2}=r5~(FMd2`9o<>)pY^^uhicw; z)0*EJYd*f|2W8vVcZhfVdvC-d-(R+QGIcK;>Cl3m(ionOLx2T zGCpA4bzUY`IxkZTmj6!oIxjP4otJrQVZ++7yuGw!+1~Pf=VkS&^RhP5c{%hsM~9u4 zlS}KN^KyFAc{#uByj)FoUT*vA!uq}Qa_{~&Wq=XFPW zv(BqOps)T+zWNz>^^^JPTkZABspU3K@4SBfYE4-4*1mOMxs6jeuTJ3@`?M~s$Ik1p z+Yj6Bu-kjQ_By;^Em}*K`xy55!*1^s&8t%}ufslPWXu}3)~!uz+w!qd+Zr9RMl7E{ zx?#D$QTsdkSLbzX*m6H(H#pZ*xrnf z%{b zZD-flcGrFFEn7Z!&wcM*TDGz0_Itio`|f|=eeXLi`}TR?=lhlLx^JKMeQ)g_yS2;IB6hg%i-PcMkQvZQi@Td*pi`zyG`Q`hfiKXXo|Nem?G4w(;omAMMYh&wun~ ze6*i_E4=y*@apW-tFue5f84%}+LRMauTCbtI+6711k$S$IIm95ygDiK>SWBTlPIrF zqP#j$^6Es%tCJ$HPG-D1Ve$Hx%{n3R>cqpVlMAm-8oYW!{_2VNt0&p7o@BpzGX3g_ z^y_!q^BdyTlipWPW?wytef4DY)f3QHPb6QTF+E9qeRg=C|LDAV67lxidgpE6zVkLX zV9j^lJo(aWjd~B>|%~>0rx9);<-Ffp<{??nayuW8(dVgC_owsrO zH2%HwHeownkLGQ{{ZIVWd7B)v?9b#$=WWV3p^?)H90-_{+Wbw_=j zZo`q?u-OfJw&93wxSLITu<3SNx0a*7?d|O?>$JaK|DJmJd(||3=WPdj*OznGC+ymg zALzIJ`Oe$HVCU`7t{?e?V{ae3gHsu#QmS(w+mm+i$}|0 zxO9h?zKoZ4;?hoB+A)`L-mWGrpL1n9e(K)7dHdU4=gr^#x9g+M+YRW=bLZ{W0lIbD zTZiw~eLL6wcIR{MAn)A%&i>!q_j_M9FYotu?|$zcEa%?e9^H@2KyQ!U|7c&FAAfT` z`^|aXH|J#E{&b-IEq!xt^3D0kH|HMToL78vj_%Dlxi>#LZ_c;9InVUwe9GJNnf26p z|833MwEpP44|J_F>znmxeeb;c!}{*Y*t^G7??c{pedXOB)c3Dr)@A2?c)j!P1jhTw zbm!f-`nzxS_t9hP)UvJ7d+Wja%X;p-I}`Cf=5}MYHRiU?MkpdW@6I;7`%!pzcH!Mw zh4-F+`w@5_w;g|&-Y5R-yib0yeEy{Sp6po%)=lSq%KrP~^FHnUzV+W{+-Amp`SyRG zx$eBr+TU3pn_cd_&oR#V*xX^~ectEHd*A$)wQt$)dG>kRoWHdkgL(Ts?>H^kUoTt# zzTo4F?t5{z^X|vteVJu>!?KN)LCbxv@UA#^tB%hq^P2ngM@jQx9a-M*2jP8vz_RcD zK)rA9Zn&L4Qtz8?v*q?%w(lGMeaG?LaV&Sot(ngIp4;ttd(Xb@pIVNM3!nY(|95|B z-Vaz0?AO7fW#10m$ANu6urG)1^Kinl&BH0nz8wB;`P?Jh^;hZr$bB6f9jZ?+0Qedd**BB=jQ#=ZLZw@$}zoi|KDuSU!3=A`+sdeZyc{1@4NNi zTl;nESln{@Ie7Qj^Zm}}-r46n<~v{id&lzLF}QcT2itk@xsSfCkM_g0r+1G{-(9bH z_gM4Y^_O?&f8U+seRr<*-TBk^@Ak`i(|6}b-<=PAcb@azpM>`(Ul-@~KAdy<`0cAT zZp~V^oez(}Kb*b!7`U>&Sx?q;=VNfp>RA`oP3Ob)%a1QUHnME(S;y9?<>Mo^GxB%m zW7PJY%=>T>?_<>M{9JsDZgoDKb^CDE?Zbn<4`TZCKZqZFIkPK6(e% zVdrDq`^VkS_(SJo!oE(-S^hog{`8<9Q})jTv=0x^KBm^KBg=hwVD>TX<1;=!v()*R zbKi5`H)s2E_Hll(^Rcks`B<8;91{;z?SG<2^+Oo|R+w{Htv2xb=@XP&U z)!S>kosacNi+{uWH&`}pW7D>_b~+#1j)7m&A0A+R?AW&*$9Bhl?+sd0mVMgWwA^;j zvDou@`|j_+=N;Iu1IO6e!jD78=g__$*|#H~e{6fFwsq>5pW08~iyvn`cIGx`_WjJZ z&V25@x8GRH%vDiw?6OIKDs3T;gbBv zt>b;?zkXML+}p2v-y426e>^yj500bX%^#1B_oMf_7WLub%!lheAFgS9xQ_ARn#6}| z5g#6|d^oTE;e7gsbKxJZ(SJDq{rTIT^?T>j@83^n3qJ>LtiL;-gHzUwHQ)Jkg5}d$ zfKShBKZm||K3xm{{Ib^h^n3WzlbcW1p+7xA`E-r=b7a^Ww|xGH+j-dWIcoc3?rY3; zdrQ`G=X3l^=X1h+PW{pOoL=aBE z*0$yT{qFnRdhUGsKKk7LYWeuK{oZ!l?Y}yo9y+dT96u8V)3A6T|=Za+{Im1v+VrPusGlC*e!7PF`KN8UR{8m-eRS>c({;g5*9AZQ zP5yMv?ej06@1gCd>ujH{qkZ}t{psQD=im0#^|4RiE1#}SeYzI)>AKUWYdD{-;e5It z^XWRwr|U4ylb*IZ)@kcUowjk+X`7RscK+CD7qix(<^A7&vj(g|YsmUyeYJ+I5o^@4 zjc;A6XN_AEmTh?))4okx?&q6rew(xAtp#h*TC$d{6>HU6v(~K*Yt!1Y?B}-~YuDPd z_N@cU{(bB3-?4RKomywsxpiS(T1?-*S=ZK$b!**O_tt~O>X}XZ_J{SS^_TUx_1${1 zo;%IgtX=b8^I!8{^I!A(`ES?!eh0K`{%ihg{%ihgevjMQHUBmLHUBmLHUBlg-x=+i z-}geh=D+5@=D+5@=D+5@=D+6m_pe>^d)C#i`91S$o_V!vevb>=HUBmLHUBmLHUBlg zXJ+l1|C--3w06z!?`XT`zvjQ@zvlP5t6lS7^ZOgy{NDk#YyNBgYyKPl8~z*q8~z*q z8~z*q8-9;V+YSE>{|)~Qzh{H(hX02DhX01&-|lw9f5U&nf5U&nf5U&nf5U&nf5Y#0 zX}jUS;lJVcd$ry0-|*k?-|*k?-|*k?`<>fv`2F5(H~csJH~csJH~csJH~csJH~csJ zH~csJH~csJH~gN9v>W~#evdPn$J@=b+U9q8^Siv=^562`^563N-QN6eZ@2tLu9}gn z=J9#+_`EqM)^7Q4`EU6>W@-Kgv|IjLeve<;E&nb5E&nb5E&nb5E&nb5E&nb5E&naQ z$2;wo-|vrh%kQyI^Vp}|^562`^56117uRn2Z~1TeZ~1TeZ~1TeZ~1TeZ}~lLYPbBi z{I~qK{C+pJTmD;qkEz-nzqy`v$A8Cv$A8Cv$M0`iyW_v(zvI8-zvI8-zvK6Ktljb7 z@!#>^@!#=#-qOqnwL5-i4cZ;Qzm3hzP%|^sJfCTI{CE6!{2tS_JN`R<^F-~A|BnBT z|BnBT|BnBT|BnBT-($db$A8Cv$L}0!^Bk$&@!#>^@!#>^@!#>^@!#>^@p~TC?)dNd zJ)deGOEzaV+CBe0|2_Xb|2_Xbzfrqp)UFw|Yxn&3{P+CMgtU8pFGtz#`S1Df`JE?i z_x$(#_x$(#9;>!{{(Js={(Js=e%CkKJ-^4X&0~jl&wtPF??>~0XV5&?Yo6;hXI$Dn z|2@AmFYTV+89!e=3IXBe5pB~-^}VYvwF>}UNfuL%<47IqndO4%{l&NR}UNfuL%<46>dhK_9vwF>}UNfuL%<47Q1e$XZ&8%Mgo!_ip^LVKJ&Tm$) znbm7%^_p3|W>&A6)oZRHG_!im8N+5)ubI_rX7!p`z2>??GppB}+h}I>npwT(Izuz7 z*UaiQvwF=nhvvCobKRl2?$FHYHM4rntX^{lvYFLuu0u4ldd>60W>&A6)oW(;n&*nm ztX?y#*UaiQvwF>}UNfuL%<8p2_|57yvwF>}UNfuL%<46>dd;j}GppCk>NT@^&8%K? zouir6Yi9MDS-s|(M>DI}%<46>dd;j}GppCk>NT@^&8%KCtJloxHM4r{4}PL8S-oafuelb}T#IRD_1d5OX7!p`y=GRgxjxg(>NT@^&8%KCtJlox zHM4rntX^~NrkT}iX7!rKxy`IzGppDBF4J8 zT64~(|WsHM4%rtY0(h*Ub7gvwqF2Uo-31%=$I6e$Ba_X4bEHUfs<4HM4%rbEan2 zubK60&I2{Ge$A|3GwavP`Zdp|npwYQ)~|i%|IYuN-^^e0Jgb@cYi9nMnZNd(|2w}+ z4bAza<~dk%$)TD3Yi9r2cYd>f?K{8uzxJKq3}Ex}-pw3f`_69`u(^2IT&8HA_cybE z%`9Lu3)sv8HnV`uEMRl_qL~G3E@3pYfXysmbB?xs=Qj)3zVkbu)m+|aW&+!Hesh8C zJHOe$<~&z(o~wQ5HzU~mf1|eV{NMRqIBn(yn|Z-zUa*-LY|e)@7gL*g!De2tnHOx% zk2RN0noB3mbIsAn!T0_#H?vjaZ}BnANp<*0?ocO)v{>97ziGb= z@SFI{0Kcif4Dg%$%K*RWzYOr30L%ctDZmWyJC~dRe$#*%;2+>O6_^2jlYtrFcfKzJ z{I0uXfPa90fZsXjI0qe5gK<7ECI@43FfLcd^k7_{$pF6z!We+a0KapFam^;qTgQ3p z4D!3QnL++R{z3jh{z3jh{y~1{wqw>X&TnUse~^EW-?_&O@;m>SLHq^W{`i7e~^EWe~{k@V$2$5kbjVWklzqt3=zh_Uxfqy>nYfsViwUckw~8^i7=w$Ex0uO_k+&Fm%Miaow+!(Q@f&oD zLAMO?yVM)kv|_X^X4+zyEkpc6{LZyzh~MB^%&Nt7S4^a3h~K1HjF4rB-=*TXCKf{{ zG3gcKTrtEI=V@c)D#oeeQgTd7#avHJOT}bVOh(0!O3XvW+*8a$#gJ1BImK{ShWJf1 z#T-+d?~OU8m}ANi{}BHWzj>yZXNq~Im}iQ4rkH1nbHg#ylp%i2RWw)8T*WnuXs+^w zUvm}BRb1kZ<|<$KHCNGGMROI+RWw)8T*Z0jXs+^wUw0MVRle|l;s3(_ga(-O9%vXNRS6teU^XK`> z|CQfm{y3K&m-_RS|11Aj{;&LB`3(@|EB{x1tyr{T(Te3O|5yI6{9pOM@_*$wE)We_ zG-T0`#qd!KA4NkJ14uDS5CceY?mmW)q9Kbxq`0mc4Oui~(U3($77bZ6WYLi2E5Bi- zeC7YjZyX_C`M>gi<^Rh6mESN^hWQONWtd-k7VTMv`G@&6Xc^`o<{#!C<~R5hqYWA6 zH{K9~>oEuwgX__$#XwXHM8!Z%Gb-Y%~HMRS)C{tj7)UgDF$k6s{t%HG2>%HG2*2x&8Q~w{AK^EG64xPPmM{j< z;(BC8`A7K;rNwwkga#SqALVzgGNb&X{Gwt{=zs zK{G5#_BF@7@@QE;To-{n_wM9GmZf0tj;5krUR^1B9|E`OK5%irbi^1Cjb zE`OK5%Wp;_UH&eAm%q#3GF5^yZl}LF2AA1bosmdUH&eAm%q#3HSZ)8p^)_xKevQOra!6U9t={5}32e~-V%-{bG`8~#j>zsKL> zHwYSopy~0Omr0M`*jRe}3>n7N3Vix`}Z5xh+BPw-Fh zo5_x0+Dz~psLcfb1pfs81pfs81iyKWOz=K^#Px2e)jd8)4pUNcvB)>VT zO!Awh$|S#es!Z}v@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@*7Xg zB>yD;B>yD8nXOFnPx4Rl8(+*M|0Mq;znPXy@=x+l@=x-c@yaCsB>yD8!Q@QxPx4Rl zPx4RlPx70G$t3?I{}lfe{}jJL<`|QVG09BvPw^XUj@g-*oyiox`I$`dPw`LjPw`Lj zPw`LjPw|_-&lLX@{}jJj{7msr@teKL6u)`27?jQw{}lfezoF?E&y2z8O!1r1$rS$- z{}jJr>P+!Z@lWxa-^moe8J8?eq4zt?`x6#o?e6u;SxOz}_gPw`LjPw`LjPxDXn zPxDXnPxDXno0ZEnzhUi|naec4o*>iwhPN}#Kg~bQKg~bQKg~bQZ%!oB{L}o?{L}o? z{L}nKPh;pi)BMx?)BMx?X7V!4Kg~bQKg~bQKg~bQKg~bQKg~bQKg~bQKg~bQKg~bQ zKg~bQKg~bQZ@4^$%QMYC%|Fd=-YL`k)BH31GyF6BGyF6BGyF6BW&|_CZ#F73{4@MB z{4@MB{4@MB{4@MUX*0t=!#~46!#~46!#~46!#~4s<}fq-GyF6BGyF6BGyF6BGyF6B zGyF6BhT&sIF*E#Hj?D1S@Xzqi@Xzp@U(5{u4F3%O4F3$jvEIz^&+yOi&+yOi&+r@f z%?$r6|1AG3|1AG3zgfu4^3U?m@|%myEdMOO0p85=&+^am&+^am&+^am&+^am&+^am zo2SeyztQ5%^3U?m@*D8aEdMP3EWcs@%<|9j&+^am&+^am&+^am&+^am&+^am&+^am z&+^am&+^am&+^am&+^am>vl5BuieQkzo;Oy{ImRX{B!(s{B!(9pEJil$3MqE$3Mq! zCNy*WbNqAsbNqAsbNqAsbNqAsbNqAsbNqAsbNqAsbNqAs=1MciKgU1EKgU1EKgU1E zKgU1EZ}v2E{B!(s{B!(s{B!(s{O0sB$3Mq!UN3X}X7)12KgTbc$Q=J1{~Z4u{~W)W zzRdB@@z3$k^Uw1ua5K+8&p*#^?ltrL^ZfJt^ZfJt^ZfJt^ZfJt^ZfJt^ZfJt^ZeSV zm>bMIzuDQ$^Uw3o^Uw3o^Uw2}sm(n9JpVlZJii&+%=6Fl&-0tN%{>1+zm6&M{PX2h33;YZG z3;epQEbuSzFYuco&I11e{{sI4{{sI4zcKtQ@GtN$@GtN$@GtN$@GtN$@GtP2)yo3^ z0{;U40{;U40{;U40>4?lEb=e%FY=3OvdF*4Z`41F{EPgH{AT{L$iK+H$iK+H$iK*M z);Wv(i~NiHi~KTyEb>bQvdF*4zsN7v$s+$E|04e)|04e)zoa0G{EPgH{EPgH{EPgH z{EPhN4ztL=$S*(0BL5=4_$Q0}i~J&>m`%(gzdRv}{EPgH{EPgH{EPfc{7d{|p)Bz) z@k<%9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9 z#J|M9#J|LEPCT-SEb%Y#FYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz& zFYzz&FYzz&FYzz&FYzz&FY_<+FY_<+FY_<+oAb>w|1$qF|1$qF|1$qF|1$qFzuERI z^P35d&LhkG%lymy%lymy%lymy%lu|VV-`Nk{N~}K3&}G7GXFCFGXFCFGXFCFGXFCF zGXFCFGXFBadDATOFY_<+FY_<+FZ1gfv&_HDzs$ePzs$ePzs$ePzszrbIV=1t{O0nr z!oR}5!oR}5!oR{VSjh^%`PQuPn{mww{|dj^=a{q13jYfK3jYfK3jYfK3jYfK3jYef zIq9tMukf$%ukf$%ukf$%ukf$%ukf3vj%Y1rs9L%D>9L%D>9L%D>9L$}jfGD*r0~D*r0~D*r0~D*r0~D*r0~ zD*r0KIr6OXukx?*uk!0AvdX{8zskSLzsfHp$|}Ek&dl@vrf(@vrf(@r#kN#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD z#=pkD#=pkD#=pkD#=pkD#%~rnW}&mjzsA4DuLFtM=&bRt@vrf(@vrggL$by%n93Ue z8vh!YldSWv^RM%-^RM%-^9!&dz{)!R zI{!NVI=`9ztn;t)uk)|-uk)|-uk#BlBihP3|2qFVzkn<2{OkOhmS|eC&M)f9I=|WP ztn;t)uk)K*j&309{OkPd{OkPd{OkPd{G!dQ^P3sZI{!NVI{!NVI{!NVI{!NVI{!NV zI{!NVI{ya$2LA^C2LA^C2LA^C2ERGZ%#cM{DQP>@Ne*& zTh9jn2LA^C2LA^C2LA@Xnf7e(Z}4yMn{Uqs{|5gC{|5gC{|3Lg_iXU%C$ho6!N0+8 zraK$_8~hvm8~hvm8~hvm8~hvmnu~1ki{$)&!C3qo{2Tn6{QC85@^A8Q@(byr(a0wM zCjTb?Cco||oBW&nX34Y3zsbMJzsbMJzsbMJzsWDg%O?LO|0e$?|0e$?|0e$?|0e$? z|0e$?|0e$?|0ci2DVzM8{G0rn{G0rn{G0rn{G0rn{G0rn{G0rn{G0rn{G0rn{CbmY z@^A8Q@^A8Q@^A5P@o({O@o({O@o({O@o({O@o({K53N@bB>N@bB>N@bB>N@bBY8@9^*N>r%4AuXoB0{|>+IDLec-{5$+R{5$+R{5$+R{5$+R z{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{JZ>G*zEG}^6&ENVzbM? z%fHLN%fHLN%daoYF28OzyZpQSyZpQSyZpQSyZpQSyZpQSyZpQSyZpQSyZpQSyZpQS zyZpQSyZpQSyZpQSyZqXn?DFsO@AB{R@AB{R@AB{R@AB{R3-YtezstYNzstYNzstYN zzstYNzstYNzstYJzsIjP%O3w8{~rGy{~rGy{~o`dGJE`c{CoU+{CoU+{CoU+{CoU+ z{CoU+{Mxnb@$d2P@oSK>$G^wF$FE1q9>2CNx}@y!@A2>P@A2>P@A2>P@A2>P@A2>P z@A2!H;spn?$L~c4vd6#2zsJAF??nirnadu(ZZ2BT=$^92zsJAFzt6wV@5KtT&%e*V z&+ml`vd_QIzt6wVzt6wVzt6wVzt6A5%Rc`;|31HtI$rD``~3U-`~3U-Ui={Y{9XVd z`~3U-`~3U-`}|%MA^ZIM{QLa-{9Ys>8m(xwqG8QG|33dd|33dd|33dd|33dd|33dd z|33dd|33ddzy3A*{2JKo^Y8QT^B?eQ38Vea0sjI20sjI20sjI20sjI20sjI20sjI2 z0l$VeUbP_y{0IDAy&(tuy4oD@AMhXWAMhXWAMopKbHIPVf53mh?-d_%z^`eH4mSt< z2mA;82mA;82mA;82mA;82mA;82mA;8UNIsE{0IC8{0IC8{0IC8{0IC8{0IEH$!L9Z z$glU!A^#!&A-`8(i>5M%{D=I9{D=G+%N+6_@@s>m4bCC|A^#!&A^#!&A-|?Bhx~{9 z+PWO_AMzjaAMzjaAMzjaAMzjaYwvT&f5?BxugA+F{~`Y&{~^CVFNge^6p5&seY5&seY5&seY5&seY5&seY5x=%E zNBlb39P@j!Necf6jl- zf6jl-f6njq&T`Iw&VSB-&VSB-&hK?0;`LN=&VSB-&abb|IsZAoS2)T!|2h9T|2h9T z|2e+~JLml8{95dIy|SG1pYxydYn^k>ukX(}zt=O5UOQgXJYF>^=lmD^7yK9e7yK9e z7yK9e7yK9e7yK9eUY@Oxc>T<~kebHRVX zf5Csjf5Csjf5GpyWOBiO!GFQ8&yGGj7yK9e7yK9e7yK9e7yO#@T<~A;U+`b>U+`b> zU+{ZPnq2UEU7B3*U+`b>U+`b>U+{a)gIw}o@?Y{_@_Y5DT=HM?U-Dn_dtHQF@?Y{_ z@?Y|MJ&#=SU-Dn_U-J9EF32VSCBIjo$|e6L|0Vw=|0TcIy~`!P*UHHy|0Vw=|0Vw= z|0Vw=|0Vw=|0Vw=|0Vw=|0TcI*vTdTCI2P=CBN6)$tC|K|0Vw=|0Vw=zt`f4*W$?~ z|0Vw=|0TcIBgqy26~9-@$`$_=ztc^p75^3g75^3g75^3g75^3g75^3g75^3g75^3g75^3g z6~9;W$`$_={}ulgzgPFl75^3g75^2#SNqBp{}ulg{}ulg{}ulgzy3e^|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|M&kdZ1L;=qyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2%0L4N&z>HkarU;6*j z|JNYD{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S$o`1Sv#|1bT2>HkarUqk%*|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6**@^|^W{9XPof0w_@-{tS}clo>gUH&eAm%q#3gU4H$4 z>Hn+C-{tS}clo>gUH&eAm%q#3Hkar zU;6**@%Q+9{5^jCf9e0L$FKjd9)FL&$KT`c@%Q+9{5}32e~-V%-{bG`_xOAKJ^mhl zkH5#?@A3Eed;C5A9)FL&$KT`c@%Q+9{5}32e~-V%-{bG` z_xOAKJ^pe2asF}sasF}sasF}sasF|B{eS8IOaEWv{Nw!N{Nw!k|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarUlaTj{1f~W{1f~W{1f~W{1f~W{QCdW|JMZn1pfs8 z1i${j^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2%`e^uO1kKBi0pYixu9@X9ow38f@LbsmUYYyq5mlz1p z9tv7wB*wDn@Q;-kC=jVg9Jn(@-6D1a#IYobl(pPVV>bre!q97=-$I3^Me%%#x%A%8 zkW%9u_}iJ~%+7pwm*jG0o=^J!^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|L;TlKeS)}pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>Hm9d|6}|0|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`2b_2Qo;3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K; z2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L& zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}E z*?(pKmHk)tV*tVcbY=gQ{a5y1*?(pKmHk)tU)g_U|CRk$_Fvh5W&f4^SN31oe`Wub z{TP5S0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_#E9 ze`fzP`=8m50SE&S1|SST&+LC@|1a0CZ#jjr}+F-`Ia+|Bd}O_TSilWB-l)H}>Dye`EiR{WtdC*neaHjr}+F-`Ia+ z|Bd}O_TSilWB-l)H}>Dye`EiR{WtdC*pC4S0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaPpDf2*yd0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaN1#ei}eDfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz8K* z`;!I`4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOhm1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1( zqXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}H zXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMK) z_w1(uj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}H{*C=KfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4Pf84p9U}*z-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaM`S_R|1H0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAVUwEt-T(f*_TG=R|nMgtfPU`P9p_8;v(+JChFX#dgvqy0zw zkM_6Fmvj1fN$^Mi5C;LzK zpX@)`f3p8%|H=N7{U`fR_MhxO*?+SCWdF(jll>?APxhbeKiPk>|78Ek{*(PQfYAU( z0~ifpG=R|nMgtfPU^IZy0CuwfWIqjHG=R|nMg!Q%{*(PD`%m_t>_6Fmvj1fN$^Mi5 zC;LzKpX@)`f3p8%KMi0sfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR*nhB}1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0Q-;j(*Q;T7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!Bb3@BL%z@X;@wzvC%Ghd;X5C7BOI^m3a`T2Xjb#MRX%wPWEnIHY)nIAjzNB`ADFCX&d z31==J>HnX(e5616Z(m&G#fdL2^2M3UMZA2+!Drz6UclqvGw^*c-EsKyUpl{k8T!7T zz#ZJu`+lB1zwaN$^ZN{d^ZNqj^ZPQp^ZR;~^9TMXa{jx18#;feQ#e2N3WD=v567M#d)({% z*n>q5?#p8jyEy!bga6o`ANyC}{8$Knek?aTKh`guAOG0b@3EH*pC5az)%md(&zv87 z1uBkBM%OpKl0Pc`6GY)&mU0<&L3$3&L4SY`uQU-F>!F?PdveYe&X3u zhyUW>xk-o1DUb4>pLl5E{KU^5=O_NbIzRC*!TE`D_WVSWc7E~~ZsUnp%%7il+4uR0 z*T|fodSd7N)Uy=lr=Bi2KlNDe`KbpN&rki2@BGw%Y0gjm410cxws3IQo?iQSPcL`v zsTYc$pL$jA`KgyRou7Ig%lTtZB%eR_Xw&&)|NNgnR=uA;)&!qF_DcBk$6jJ~{@ClD z&Oh`6r-M(}5B*hm{-Mm{{6jDJJ%8fa%kw9mS~`E?IhXS%9=JJw;(z1kPy9%H{^Z(s z;gu(^&sU!1Jzsem=X~Yy`}36t@Xl9$8vK?0om=1j(yf2=oA%Kdxq zynXoXd%t)1okw5$`h&y0Z``?mc<8s^y8X@HKfHeLjaOg!7ytdc?|%8s!<~b-zx(KL z_uy(ye&ulgaPOVN+n3Y7eedvlm$!%CIXpUi?eO)(gBR!S9lmkLrToU_BYN}J?ZY<@ zzkfOP`n|&&hgWa?t6Tr@Hx6&P(Zl%%r$0H|e*1MNzV(S;SBKkvF;PCvXhvz^3`Mb}5^89Cq=YRG5)8{{OnNMGR<)2?Z;`1LI zzPQNY`A-h6_2RuQ`P0j9yg2jImv4T2__@pf#D{%({;v+tKRrDE(TnSU>gNCNmu~&b z%UAr-+jrl%fBS2`v~R!h==K|T4qvH`%MV{@Ja+|IY2VzIx~N!{yCgcjd*|`!C)+c;(kz zt{<(=;4|M}9bFaOq?cOTw)@b%lT z-8tO#$$9YF?XO?%_T>w3`|#lFcV4?|>DO-G|EfFi;N{0VeC6Jww_m?~|L&a!hX;@D zf8(w{M8EhM{QT>3_${9@AKXW|eDupba0{0=-gS8Gwm+<&@BiYn{o;?0-}4)X*ZgsR z`8odl#xGa&x5SGNa9MRZ@s-29N0&c{x9=Y=cjo0q{K@&o;c}mx{l&Xw@ZDR#`q$rk z^4EWM{lBjM{Ihre;;+B^t^fJ#{mQ9`?GgH{p{U8{p=6k td*z>8KAy`TT;bpv2Uj_`&cT%qu61xV@B5x#?>YZ%Z$I_xPY opennlp-morfologik + opennlp-subword opennlp-spellcheck opennlp-uima diff --git a/rat-excludes b/rat-excludes index 5a5d86b90c..a505ab2d60 100644 --- a/rat-excludes +++ b/rat-excludes @@ -70,3 +70,8 @@ src/main/resources/opennlp/tools/tokenize/uax29/WordBreakProperty.txt src/main/resources/opennlp/tools/tokenize/uax29/ExtendedPictographic.txt src/main/resources/opennlp/tools/util/normalizer/confusables.txt src/test/resources/opennlp/tools/tokenize/uax29/WordBreakTest.txt + + +src/test/resources/opennlp/subword/sentencepiece/*.model +src/test/resources/opennlp/subword/sentencepiece/*.fixtures.tsv +src/test/resources/opennlp/subword/sentencepiece/corpus.txt From fe4abddf7b0822bdf6c083899f7d1dc11ebd97c2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 10 Jul 2026 19:08:45 -0400 Subject: [PATCH 02/82] OPENNLP-1885: Speed up the encode path 2.3x, parity-checked at every step The vocabulary trie dispatches wide nodes (the root and first level of a real vocabulary) through a 256-entry direct table, one load per byte, and scans narrow nodes' short label slices linearly instead of binary searching; a randomized differential test holds both layouts against a map-backed reference, and moving the duplicate-piece detection into the counting pass fixes the index error it previously produced. Non-unknown segments reuse the vocabulary's piece string instead of decoding their bytes, since the trie match means the bytes are identical. The normalizer precomputes, per possible first byte, whether any character-map rule or user-defined symbol starts with it; a clear bit proves the prefix machinery would pass the byte through raw, so plain ASCII text skips it entirely. The per-chunk record became a per-call scratch, the input view keeps its oversized buffers with an explicit length instead of trimming (pure-ASCII text gets an identity offset map and no map array at all), the Viterbi scratch is one interleaved array with scores as raw float bits, and the character-map trie walk relies on the JVM's own bounds checks with the fail-loud translation on the cold path. All 37 bundled parity tests and the T5-small and ALBERT real-model fixtures pass byte-identically. Single-thread throughput on the T5-small vocabulary goes from 2.83M to 6.47M pieces per second, from 0.62x to 1.42x of the reference implementation measured through its Python binding. --- .../subword/sentencepiece/BpeEncoder.java | 21 +-- .../subword/sentencepiece/ByteBuilder.java | 5 + .../sentencepiece/DoubleArrayTrie.java | 54 ++++--- .../subword/sentencepiece/IntBuilder.java | 5 + .../subword/sentencepiece/PieceTrie.java | 59 ++++++-- .../SentencePieceNormalizer.java | 132 ++++++++++++------ .../sentencepiece/SentencePieceTokenizer.java | 26 ++-- .../subword/sentencepiece/UnigramEncoder.java | 48 ++++--- .../subword/sentencepiece/Utf8Text.java | 33 ++++- .../subword/sentencepiece/PieceTrieTest.java | 116 +++++++++++++++ 10 files changed, 380 insertions(+), 119 deletions(-) create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java index 3cea8edb64..e7430032ae 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java @@ -75,29 +75,30 @@ private record Pair(int left, int right, float score, int size) { /** * Segments normalized text. * - * @param normalized The normalized UTF-8 bytes; must not be null. + * @param normalized The buffer holding the normalized UTF-8 bytes; must not be null. + * @param size The number of valid bytes in {@code normalized}. * @return The segments covering all bytes, in text order. */ - List encode(byte[] normalized) { - if (normalized.length == 0) { + List encode(byte[] normalized, int size) { + if (size == 0) { return List.of(); } // The symbol list as index-linked ranges of the normalized bytes; merged-away symbols // become empty ranges. - final IntBuilder fromB = new IntBuilder(normalized.length); - final IntBuilder toB = new IntBuilder(normalized.length); + final IntBuilder fromB = new IntBuilder(size); + final IntBuilder toB = new IntBuilder(size); final List freezeList = new ArrayList<>(); int position = 0; - while (position < normalized.length) { + while (position < size) { int matched = 0; if (userDefinedMatcher != null) { - matched = longestUserDefinedMatch(normalized, position); + matched = longestUserDefinedMatch(normalized, size, position); } final boolean frozen = matched > 0; final int length = frozen ? matched : Math.min(SentencePieceNormalizer.utf8Length(normalized[position]), - normalized.length - position); + size - position); fromB.append(position); toB.append(position + length); freezeList.add(frozen); @@ -199,10 +200,10 @@ private int resegment(String piece, int consumed, int depth, Map>> 8) & 1) == 1) { - final int value = unit(nodePos) & 0x7FFFFFFF; - result = ((long) value << 32) | (i - from + 1); + for (int i = from; i < to; i++) { + final int b = key[i] & 0xFF; + nodePos ^= b; + unit = u[nodePos]; + if ((unit & 0x800000FF) != b) { + return result; + } + nodePos ^= offset(unit); + if (((unit >>> 8) & 1) == 1) { + final int value = u[nodePos] & 0x7FFFFFFF; + result = ((long) value << 32) | (i - from + 1); + } } + return result; + } catch (ArrayIndexOutOfBoundsException e) { + throw new IllegalArgumentException( + "The trie references a unit outside its " + u.length + " units.", e); } - return result; } - private int unit(int nodePos) { + /** + * Tests whether any key starts with the given byte, which is exactly whether the root has a + * transition on it; used to precompute the first-byte gate of the normalizer scan. + * + * @param b The first key byte as an unsigned value. + * @return {@code true} if some key starts with {@code b}. + */ + boolean hasTransitionFromRoot(int b) { + final int root = units[0]; + final int nodePos = offset(root) ^ b; if (nodePos < 0 || nodePos >= units.length) { - throw new IllegalArgumentException( - "The trie references unit " + nodePos + " outside its " + units.length + " units."); + return false; } - return units[nodePos]; + return (units[nodePos] & 0x800000FF) == b; } // The offset from a unit to its children, as encoded by Darts-clone. diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java index 70a2db9ac4..ec585685c1 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java @@ -53,4 +53,9 @@ void truncate(int newLength) { int[] toArray() { return Arrays.copyOf(data, length); } + + /** {@return the backing array, valid up to {@link #length()}} */ + int[] array() { + return data; + } } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java index eeb028e00b..37c2ef7912 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java @@ -24,26 +24,54 @@ * *

Encoding walks it one byte at a time ({@link #step(int, byte)}) while scanning the input, so * every piece that starts at a given input position is enumerated in one forward pass; this is the - * lattice-population step of subword segmentation. Children of a node are stored as a sorted - * label slice and found by binary search.

+ * lattice-population step of subword segmentation. This step sits in the innermost loop of the + * encoder, so wide nodes (the root and the first level of a real vocabulary) dispatch through a + * 256-entry direct table, one load per byte, and narrow nodes scan their short sorted label slice + * linearly; both layouts enumerate identical transitions.

*/ final class PieceTrie { /** The node id returned when no transition exists. */ static final int DEAD = -1; + // A node dispatches through a 256-entry slice of directPool when it has more children than + // this; below it, a linear scan of the sorted label slice wins on memory and is branch-cheap. + private static final int DIRECT_THRESHOLD = 8; + // Per node: the slice [childStart[n], childStart[n + 1]) of labels/childNodes, and the piece id - // accepted at the node, or -1. + // accepted at the node, or -1. Wide nodes additionally index directPool at directStart[n]. private final int[] childStart; private final byte[] labels; private final int[] childNodes; private final int[] values; + private final int[] directStart; + private final int[] directPool; private PieceTrie(int[] childStart, byte[] labels, int[] childNodes, int[] values) { this.childStart = childStart; this.labels = labels; this.childNodes = childNodes; this.values = values; + this.directStart = new int[values.length]; + int wide = 0; + for (int node = 0; node < values.length; node++) { + if (childStart[node + 1] - childStart[node] > DIRECT_THRESHOLD) { + directStart[node] = wide * 256; + wide++; + } else { + directStart[node] = -1; + } + } + this.directPool = new int[wide * 256]; + java.util.Arrays.fill(directPool, DEAD); + for (int node = 0; node < values.length; node++) { + final int direct = directStart[node]; + if (direct >= 0) { + for (int edge = childStart[node]; edge < childStart[node + 1]; edge++) { + directPool[direct + (labels[edge] & 0xFF)] = childNodes[edge]; + } + } + } } /** @@ -82,19 +110,14 @@ int root() { * @return The child node id, or {@link #DEAD} when no such transition exists. */ int step(int node, byte b) { - final int from = childStart[node]; + final int direct = directStart[node]; + if (direct >= 0) { + return directPool[direct + (b & 0xFF)]; + } final int to = childStart[node + 1]; - int low = from; - int high = to - 1; - while (low <= high) { - final int mid = (low + high) >>> 1; - final int c = Byte.compareUnsigned(labels[mid], b); - if (c < 0) { - low = mid + 1; - } else if (c > 0) { - high = mid - 1; - } else { - return childNodes[mid]; + for (int edge = childStart[node]; edge < to; edge++) { + if (labels[edge] == b) { + return childNodes[edge]; } } return DEAD; @@ -140,6 +163,12 @@ void count(int from, int to, int depth) { int i = from; if (i < to && pieces[order[i]].length == depth) { i++; + // A second key ending at the same depth is a duplicate; the sort made them adjacent. + if (i < to && pieces[order[i]].length == depth) { + throw new IllegalArgumentException("The piece '" + + new String(pieces[order[i]], java.nio.charset.StandardCharsets.UTF_8) + + "' is defined more than once."); + } } while (i < to) { final byte label = pieces[order[i]][depth]; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java index 4d30b425c1..d99d4b33ce 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -45,6 +45,10 @@ final class SentencePieceNormalizer { private final boolean escapeWhitespaces; private final boolean treatWhitespaceAsSuffix; private final PieceTrie userDefinedMatcher; + // For each possible first byte, whether any character-map rule or user-defined symbol starts + // with it. A clear bit proves normalizePrefix would pass the byte through raw, which lets the + // scan skip the whole prefix machinery for plain ASCII text. + private final boolean[] ruleLead = new boolean[256]; /** * Instantiates the normalizer. @@ -98,15 +102,32 @@ final class SentencePieceNormalizer { this.escapeWhitespaces = escapeWhitespaces; this.treatWhitespaceAsSuffix = treatWhitespaceAsSuffix; this.userDefinedMatcher = userDefinedMatcher; + for (int b = 0; b < 256; b++) { + final boolean charsMapLead = trie != null && trie.hasTransitionFromRoot(b); + final boolean userDefinedLead = userDefinedMatcher != null + && userDefinedMatcher.step(userDefinedMatcher.root(), (byte) b) != PieceTrie.DEAD; + ruleLead[b] = charsMapLead || userDefinedLead; + } } - /** The normalized bytes plus the normalized-byte to original-byte offset map. */ - record Normalized(byte[] bytes, int[] normToOrig) { + /** + * The normalized bytes plus the normalized-byte to original-byte offset map. The arrays are + * builder-backed and may be oversized; {@code length} bytes are valid, and the offset map + * holds {@code length + 1} entries. + */ + record Normalized(byte[] bytes, int length, int[] normToOrig) { } // One normalization step: `consumed` input bytes produced `data[from, to)`. The data array is - // the input itself (pass-through), the replacement blob, or the replacement character. - private record Chunk(byte[] data, int from, int to, int consumed) { + // the input itself (pass-through), the replacement blob, or the replacement character. One + // mutable scratch per normalize call, refilled per chunk, so the scan allocates nothing per + // code point. + private static final class Chunk { + + private byte[] data; + private int from; + private int to; + private int consumed; boolean isSingleSpace() { return to - from == 1 && data[from] == ' '; @@ -116,46 +137,61 @@ boolean isSingleSpace() { /** * Normalizes UTF-8 input. * - * @param input The well-formed UTF-8 bytes to normalize; must not be null. - * @return The normalized bytes with the offset map; {@code normToOrig.length} is always - * {@code bytes.length + 1}. + * @param input The buffer holding well-formed UTF-8 bytes; must not be null. + * @param inputLength The number of valid bytes in {@code input}. + * @return The normalized bytes with the offset map; the arrays are builder-backed, valid for + * {@code length} bytes and {@code length + 1} map entries. */ - Normalized normalize(byte[] input) { - final ByteBuilder normalized = new ByteBuilder(input.length + (input.length >> 1) + 4); - final IntBuilder normToOrig = new IntBuilder(input.length + (input.length >> 1) + 5); + Normalized normalize(byte[] input, int inputLength) { + final ByteBuilder normalized = new ByteBuilder(inputLength + (inputLength >> 1) + 4); + final IntBuilder normToOrig = new IntBuilder(inputLength + (inputLength >> 1) + 5); + final Chunk chunk = new Chunk(); int from = 0; int consumed = 0; // Ignores heading whitespace. if (removeExtraWhitespaces) { - while (from < input.length) { - final Chunk p = normalizePrefix(input, from); - if (!p.isSingleSpace()) { + while (from < inputLength) { + normalizePrefix(input, inputLength, from, chunk); + if (!chunk.isSingleSpace()) { break; } - from += p.consumed(); - consumed += p.consumed(); + from += chunk.consumed; + consumed += chunk.consumed; } } // All input was whitespace. - if (from >= input.length) { - return new Normalized(new byte[0], new int[] {consumed}); + if (from >= inputLength) { + normToOrig.append(consumed); + return new Normalized(normalized.array(), 0, normToOrig.array()); } - final byte[] spaceSymbol = escapeWhitespaces ? SPACE_SYMBOL : new byte[] {' '}; + final byte[] spaceSymbol = escapeWhitespaces ? SPACE_SYMBOL : SINGLE_SPACE; if (!treatWhitespaceAsSuffix && addDummyPrefix) { appendSpace(normalized, normToOrig, spaceSymbol, consumed); } boolean isPrevSpace = removeExtraWhitespaces; - while (from < input.length) { - final Chunk p = normalizePrefix(input, from); - int spFrom = p.from(); - final int spTo = p.to(); - final byte[] spData = p.data(); + while (from < inputLength) { + final int lead = input[from] & 0xFF; + // Fast path: an ASCII byte no rule starts with passes through raw; the chunk would be + // the byte itself, no leading-space stripping applies, and it does not end in a space. + if (lead < 0x80 && lead != ' ' && !ruleLead[lead]) { + normalized.append(input[from]); + normToOrig.append(consumed); + consumed++; + from++; + isPrevSpace = false; + continue; + } + + normalizePrefix(input, inputLength, from, chunk); + int spFrom = chunk.from; + final int spTo = chunk.to; + final byte[] spData = chunk.data; // Removes heading spaces in the chunk if the previous chunk ended with whitespace. while (isPrevSpace && spFrom < spTo && spData[spFrom] == ' ') { @@ -174,8 +210,8 @@ Normalized normalize(byte[] input) { isPrevSpace = spData[spTo - 1] == ' '; } - consumed += p.consumed(); - from += p.consumed(); + consumed += chunk.consumed; + from += chunk.consumed; if (!removeExtraWhitespaces) { isPrevSpace = false; } @@ -200,9 +236,11 @@ Normalized normalize(byte[] input) { throw new IllegalStateException("The offset map has " + normToOrig.length() + " entries for " + normalized.length() + " normalized bytes."); } - return new Normalized(normalized.toArray(), normToOrig.toArray()); + return new Normalized(normalized.array(), normalized.length(), normToOrig.array()); } + private static final byte[] SINGLE_SPACE = {' '}; + private static void appendSpace(ByteBuilder normalized, IntBuilder normToOrig, byte[] spaceSymbol, int consumed) { normalized.append(spaceSymbol, 0, spaceSymbol.length); @@ -211,19 +249,24 @@ private static void appendSpace(ByteBuilder normalized, IntBuilder normToOrig, } } - // Normalizes the longest applicable prefix of input[from, ...): a user-defined symbol passes - // through raw, otherwise the longest character-map rule applies, otherwise one code point - // passes through raw (or becomes U+FFFD when the lead byte is malformed). - private Chunk normalizePrefix(byte[] input, int from) { + // Fills the scratch with the normalized form of the longest applicable prefix of + // input[from, inputLength): a user-defined symbol passes through raw, otherwise the longest + // character-map rule applies, otherwise one code point passes through raw (or becomes U+FFFD + // when the lead byte is malformed). + private void normalizePrefix(byte[] input, int inputLength, int from, Chunk chunk) { if (userDefinedMatcher != null) { - final int matched = longestUserDefinedMatch(input, from); + final int matched = longestUserDefinedMatch(input, inputLength, from); if (matched > 0) { - return new Chunk(input, from, from + matched, matched); + chunk.data = input; + chunk.from = from; + chunk.to = from + matched; + chunk.consumed = matched; + return; } } if (trie != null) { - final long match = trie.longestPrefixMatch(input, from, input.length); + final long match = trie.longestPrefixMatch(input, from, inputLength); if (match >= 0) { final int value = (int) (match >>> 32); final int length = (int) (match & 0xFFFFFFFFL); @@ -233,22 +276,33 @@ private Chunk normalizePrefix(byte[] input, int from) { while (blob[replacementTo] != 0) { replacementTo++; } - return new Chunk(blob, replacementFrom, replacementTo, length); + chunk.data = blob; + chunk.from = replacementFrom; + chunk.to = replacementTo; + chunk.consumed = length; + return; } } } - final int charLength = Math.min(utf8Length(input[from]), input.length - from); + final int charLength = Math.min(utf8Length(input[from]), inputLength - from); if (isMalformed(input, from, charLength)) { - return new Chunk(REPLACEMENT_CHAR, 0, REPLACEMENT_CHAR.length, 1); + chunk.data = REPLACEMENT_CHAR; + chunk.from = 0; + chunk.to = REPLACEMENT_CHAR.length; + chunk.consumed = 1; + return; } - return new Chunk(input, from, from + charLength, charLength); + chunk.data = input; + chunk.from = from; + chunk.to = from + charLength; + chunk.consumed = charLength; } - private int longestUserDefinedMatch(byte[] input, int from) { + private int longestUserDefinedMatch(byte[] input, int inputLength, int from) { int node = userDefinedMatcher.root(); int longest = 0; - for (int i = from; i < input.length; i++) { + for (int i = from; i < inputLength; i++) { node = userDefinedMatcher.step(node, input[i]); if (node == PieceTrie.DEAD) { break; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 12e8271d76..5fa7855fde 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -253,10 +253,11 @@ public List encode(CharSequence text) { throw new IllegalArgumentException("The text must not be null."); } final Utf8Text input = Utf8Text.of(text); - final SentencePieceNormalizer.Normalized normalized = normalizer.normalize(input.bytes()); + final SentencePieceNormalizer.Normalized normalized = + normalizer.normalize(input.bytes(), input.byteLength()); final List segments = algorithm == Algorithm.UNIGRAM - ? unigramEncoder.encode(normalized.bytes()) - : bpeEncoder.encode(normalized.bytes()); + ? unigramEncoder.encode(normalized.bytes(), normalized.length()) + : bpeEncoder.encode(normalized.bytes(), normalized.length()); final List out = new ArrayList<>(segments.size()); final byte[] norm = normalized.bytes(); @@ -269,10 +270,14 @@ public List encode(CharSequence text) { int pendingUnkEnd = 0; for (final Segment segment : segments) { - final String piece = new String(norm, segment.from(), segment.to() - segment.from(), - StandardCharsets.UTF_8); final boolean isUnk = segment.id() == unkId; final boolean isControl = types[segment.id()] == TYPE_CONTROL; + // A non-unknown segment's bytes are exactly the vocabulary piece's bytes (that is what + // the match meant), so the vocabulary string is reused; only unknown segments carry + // surface content that needs decoding. + final String piece = isUnk + ? new String(norm, segment.from(), segment.to() - segment.from(), StandardCharsets.UTF_8) + : pieces[segment.id()]; if (isControl) { if (pendingUnk != null) { @@ -332,8 +337,10 @@ public AlignedText normalizeAligned(CharSequence text) { throw new IllegalArgumentException("The text must not be null."); } final Utf8Text input = Utf8Text.of(text); - final SentencePieceNormalizer.Normalized result = normalizer.normalize(input.bytes()); - final String normalized = new String(result.bytes(), StandardCharsets.UTF_8); + final SentencePieceNormalizer.Normalized result = + normalizer.normalize(input.bytes(), input.byteLength()); + final String normalized = new String(result.bytes(), 0, result.length(), + StandardCharsets.UTF_8); final int[] normToOrig = result.normToOrig(); final byte[] norm = result.bytes(); @@ -345,9 +352,10 @@ public AlignedText normalizeAligned(CharSequence text) { int groupOrigEnd = -1; int groupChars = 0; int b = 0; - while (b < norm.length) { + final int normLength = result.length(); + while (b < normLength) { final int byteLength = Math.min(SentencePieceNormalizer.utf8Length(norm[b]), - norm.length - b); + normLength - b); final int origStart = input.charOffset(normToOrig[b]); final int origEnd = input.charOffset(normToOrig[b + byteLength]); final int chars = byteLength == 4 ? 2 : 1; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java index a53ca70740..49ffcc2909 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java @@ -67,33 +67,37 @@ final class UnigramEncoder { /** * Segments normalized text. * - * @param normalized The normalized UTF-8 bytes; must not be null. + * @param normalized The buffer holding the normalized UTF-8 bytes; must not be null. + * @param size The number of valid bytes in {@code normalized}. * @return The best-path segments covering all bytes, in text order. */ - List encode(byte[] normalized) { - final int size = normalized.length; + List encode(byte[] normalized, int size) { if (size == 0) { return List.of(); } - // The best path ending at each byte position (exclusive end). - final int[] bestStartsAt = new int[size + 1]; - final float[] bestScore = new float[size + 1]; - final int[] bestId = new int[size + 1]; - java.util.Arrays.fill(bestStartsAt, -1); + // The best path ending at each byte position (exclusive end), interleaved as + // [startsAt, scoreBits, id] triples so a frontier update touches one cache line. Scores + // travel as raw float bits, a lossless round trip; all arithmetic happens on the floats. + final int[] best = new int[3 * (size + 1)]; + for (int i = 0; i <= size; i++) { + best[3 * i] = -1; + } + best[1] = Float.floatToRawIntBits(0.0f); int startsAt = 0; int maxFrontier = 0; while (startsAt < size) { - float bestScoreTillHere = bestScore[startsAt]; + float bestScoreTillHere = Float.intBitsToFloat(best[3 * startsAt + 1]); if (bestScoreTillHere < -SCORE_RESET_THRESHOLD || bestScoreTillHere > SCORE_RESET_THRESHOLD) { // Re-bases accumulated scores to keep float precision on very long inputs; every // reachable frontier position shifts by the same offset, so the argmax is unchanged. final float offset = bestScoreTillHere; for (int i = startsAt; i <= maxFrontier; i++) { - if (i == startsAt || bestStartsAt[i] != -1) { - bestScore[i] -= offset; + if (i == startsAt || best[3 * i] != -1) { + best[3 * i + 1] = Float.floatToRawIntBits( + Float.intBitsToFloat(best[3 * i + 1]) - offset); } } bestScoreTillHere = 0.0f; @@ -122,10 +126,11 @@ List encode(byte[] normalized) { // User-defined symbols receive a length bonus instead of a trained score. final float score = userDefined[id] ? 0.1f * (length - 1) : scores[id]; final float candidate = score + bestScoreTillHere; - if (bestStartsAt[keyPos] == -1 || candidate > bestScore[keyPos]) { - bestScore[keyPos] = candidate; - bestStartsAt[keyPos] = startsAt; - bestId[keyPos] = id; + final int slot = 3 * keyPos; + if (best[slot] == -1 || candidate > Float.intBitsToFloat(best[slot + 1])) { + best[slot + 1] = Float.floatToRawIntBits(candidate); + best[slot] = startsAt; + best[slot + 2] = id; } if (!hasSingleNode && length == mblen) { hasSingleNode = true; @@ -136,10 +141,11 @@ List encode(byte[] normalized) { final int end = startsAt + mblen; maxFrontier = Math.max(maxFrontier, end); final float candidate = unkScore + bestScoreTillHere; - if (bestStartsAt[end] == -1 || candidate > bestScore[end]) { - bestScore[end] = candidate; - bestStartsAt[end] = startsAt; - bestId[end] = unkId; + final int slot = 3 * end; + if (best[slot] == -1 || candidate > Float.intBitsToFloat(best[slot + 1])) { + best[slot + 1] = Float.floatToRawIntBits(candidate); + best[slot] = startsAt; + best[slot + 2] = unkId; } } @@ -149,12 +155,12 @@ List encode(byte[] normalized) { final List results = new ArrayList<>(size / 4 + 1); int endsAt = size; while (endsAt > 0) { - final int from = bestStartsAt[endsAt]; + final int from = best[3 * endsAt]; if (from < 0) { throw new IllegalStateException( "The Viterbi path is broken at normalized byte " + endsAt + "."); } - results.add(new Segment(from, endsAt, bestId[endsAt])); + results.add(new Segment(from, endsAt, best[3 * endsAt + 2])); endsAt = from; } Collections.reverse(results); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java index e331fdcc83..6ebb080f5e 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java @@ -28,11 +28,14 @@ final class Utf8Text { private final byte[] bytes; + private final int byteLength; + // Null for pure-ASCII text, where byte offsets equal UTF-16 offsets. private final int[] byteToChar; private final int charLength; - private Utf8Text(byte[] bytes, int[] byteToChar, int charLength) { + private Utf8Text(byte[] bytes, int byteLength, int[] byteToChar, int charLength) { this.bytes = bytes; + this.byteLength = byteLength; this.byteToChar = byteToChar; this.charLength = charLength; } @@ -45,6 +48,19 @@ private Utf8Text(byte[] bytes, int[] byteToChar, int charLength) { */ static Utf8Text of(CharSequence text) { final int charLength = text.length(); + // The common case: pure ASCII, where the bytes are the chars and the map is the identity. + int ascii = 0; + while (ascii < charLength && text.charAt(ascii) < 0x80) { + ascii++; + } + if (ascii == charLength) { + final byte[] exact = new byte[charLength]; + for (int i = 0; i < charLength; i++) { + exact[i] = (byte) text.charAt(i); + } + return new Utf8Text(exact, charLength, null, charLength); + } + final byte[] bytes = new byte[charLength * 3 + 1]; final int[] byteToChar = new int[charLength * 3 + 2]; int b = 0; @@ -82,16 +98,21 @@ static Utf8Text of(CharSequence text) { c += charCount; } byteToChar[b] = charLength; - final byte[] exact = java.util.Arrays.copyOf(bytes, b); - final int[] exactMap = java.util.Arrays.copyOf(byteToChar, b + 1); - return new Utf8Text(exact, exactMap, charLength); + // The oversized buffers are kept and carried with an explicit length instead of being + // trimmed; the per-call copies were pure allocation traffic. + return new Utf8Text(bytes, b, byteToChar, charLength); } - /** {@return the UTF-8 bytes} */ + /** {@return the UTF-8 buffer; valid up to {@link #byteLength()}} */ byte[] bytes() { return bytes; } + /** {@return the number of valid bytes in {@link #bytes()}} */ + int byteLength() { + return byteLength; + } + /** {@return the length of the original text in UTF-16 units} */ int charLength() { return charLength; @@ -104,6 +125,6 @@ int charLength() { * @return The UTF-16 offset; the text length for the end offset. */ int charOffset(int byteOffset) { - return byteToChar[byteOffset]; + return byteToChar == null ? byteOffset : byteToChar[byteOffset]; } } diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java new file mode 100644 index 0000000000..bbde60825c --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The trie's transition function held against a naive map-backed reference over randomized + * vocabularies, so the hybrid direct-table and linear-scan node layouts are proven to enumerate + * identical transitions; the encoder's correctness rests on that equivalence. + */ +class PieceTrieTest { + + @Test + void testStepsMatchAMapBackedReferenceOverRandomVocabularies() { + final Random random = new Random(7); + for (int round = 0; round < 20; round++) { + // Piece count crosses the direct-table threshold in both directions, so wide and narrow + // roots are both exercised. + final int pieceCount = 2 + random.nextInt(60); + final Set keys = new HashSet<>(); + while (keys.size() < pieceCount) { + final int length = 1 + random.nextInt(5); + final StringBuilder key = new StringBuilder(); + for (int i = 0; i < length; i++) { + key.append((char) ('a' + random.nextInt(random.nextBoolean() ? 26 : 4))); + } + keys.add(key.toString()); + } + final byte[][] pieces = new byte[keys.size()][]; + final int[] ids = new int[keys.size()]; + final Map reference = new HashMap<>(); + int index = 0; + for (final String key : keys) { + pieces[index] = key.getBytes(StandardCharsets.UTF_8); + ids[index] = index; + reference.put(key, index); + index++; + } + final PieceTrie trie = PieceTrie.build(pieces, ids); + + // Every walk over random query strings must accept exactly the reference's prefixes. + for (int query = 0; query < 200; query++) { + final int length = 1 + random.nextInt(8); + final StringBuilder text = new StringBuilder(); + for (int i = 0; i < length; i++) { + text.append((char) ('a' + random.nextInt(6))); + } + int node = trie.root(); + for (int i = 0; i < length; i++) { + node = trie.step(node, (byte) text.charAt(i)); + final String prefix = text.substring(0, i + 1); + final boolean anyKeyHasPrefix = + keys.stream().anyMatch(k -> k.startsWith(prefix)); + if (node == PieceTrie.DEAD) { + assertEquals(false, anyKeyHasPrefix, "dead end despite live prefix: " + prefix); + break; + } + final Integer expected = reference.get(prefix); + assertEquals(expected == null ? -1 : expected, trie.value(node), + "value mismatch at prefix: " + prefix); + } + } + } + } + + @Test + void testWideRootDispatchesAllByteValues() { + // 200 distinct single-byte pieces force the direct-table layout at the root. + final byte[][] pieces = new byte[200][]; + final int[] ids = new int[200]; + for (int i = 0; i < 200; i++) { + pieces[i] = new byte[] {(byte) (i + 20)}; + ids[i] = i; + } + final PieceTrie trie = PieceTrie.build(pieces, ids); + for (int b = 0; b < 256; b++) { + final int node = trie.step(trie.root(), (byte) b); + if (b >= 20 && b < 220) { + assertEquals(b - 20, trie.value(node)); + } else { + assertEquals(PieceTrie.DEAD, node); + } + } + } + + @Test + void testDuplicatePiecesFailLoudly() { + final byte[][] pieces = {{'a'}, {'a'}}; + assertThrows(IllegalArgumentException.class, () -> PieceTrie.build(pieces, new int[] {0, 1})); + } +} From db1c37149a3db4dd1ce39905a6e9636f78f1d77d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 11 Jul 2026 10:46:56 -0400 Subject: [PATCH 03/82] OPENNLP-1885: Move the subword contract into opennlp-api SubwordTokenizer and SubwordPiece move to opennlp.tools.tokenize, next to Tokenizer and WordpieceTokenizer, matching where every other seam of this round lives. The opennlp-subword module keeps only the SentencePiece implementation. --- .../src/main/java/opennlp/tools/tokenize}/SubwordPiece.java | 2 +- .../main/java/opennlp/tools/tokenize}/SubwordTokenizer.java | 4 ++-- .../opennlp/subword/sentencepiece/SentencePieceTokenizer.java | 4 ++-- .../subword/sentencepiece/SentencePieceAlignmentTest.java | 2 +- .../sentencepiece/SentencePieceModelValidationTest.java | 2 +- .../subword/sentencepiece/SentencePieceParityTest.java | 2 +- .../subword/sentencepiece/SentencePieceRealModelEvalTest.java | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) rename {opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword => opennlp-api/src/main/java/opennlp/tools/tokenize}/SubwordPiece.java (98%) rename {opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword => opennlp-api/src/main/java/opennlp/tools/tokenize}/SubwordTokenizer.java (96%) diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java similarity index 98% rename from opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java rename to opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java index 1791583101..7f790650cf 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package opennlp.subword; +package opennlp.tools.tokenize; import opennlp.tools.util.Span; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java similarity index 96% rename from opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java rename to opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java index ea31342bf2..5c371287f2 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package opennlp.subword; +package opennlp.tools.tokenize; import java.util.List; @@ -24,7 +24,7 @@ * *

Subword tokenization is the input layer of modern sequence models: text is decomposed into * pieces from a trained vocabulary so that any input, including words never seen in training, maps - * to a bounded id space. Unlike a linguistic {@code Tokenizer}, the segmentation is + * to a bounded id space. Unlike a linguistic {@link Tokenizer}, the segmentation is * vocabulary-driven, and the pieces are in the model's normalized form rather than substrings of * the input. The offsets carried by each {@link SubwordPiece} are what tie the two worlds * together: they always refer to the caller's original text.

diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 5fa7855fde..5b39c395a9 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -26,8 +26,8 @@ import java.util.List; import java.util.Map; -import opennlp.subword.SubwordPiece; -import opennlp.subword.SubwordTokenizer; +import opennlp.tools.tokenize.SubwordPiece; +import opennlp.tools.tokenize.SubwordTokenizer; import opennlp.tools.util.normalizer.AlignedText; import opennlp.tools.util.normalizer.Alignment; import opennlp.tools.util.normalizer.OffsetAwareNormalizer; diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java index 02815aa372..41e2a906fc 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import opennlp.subword.SubwordPiece; +import opennlp.tools.tokenize.SubwordPiece; import opennlp.tools.util.Span; import opennlp.tools.util.normalizer.AlignedText; diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java index 8f0b6c64a9..66595eff8b 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; -import opennlp.subword.SubwordPiece; +import opennlp.tools.tokenize.SubwordPiece; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java index 4485e13f16..7d970e3c5e 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java @@ -30,7 +30,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import opennlp.subword.SubwordPiece; +import opennlp.tools.tokenize.SubwordPiece; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java index e4acccd499..1255827ba1 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.api.Test; -import opennlp.subword.SubwordPiece; +import opennlp.tools.tokenize.SubwordPiece; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; From f71db4d2ee7bfbd61fe31d65d1abb0ca626d0fd8 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 09:20:57 -0400 Subject: [PATCH 04/82] OPENNLP-1885: Add WordpieceEncoder and fold the unreleased BertTokenizer into it WordpieceEncoder in opennlp-api runs the full BERT tokenization pipeline as a SubwordTokenizer: every piece carries its vocabulary id and the span of the original text, surviving the normalization steps that change, insert, and remove characters. Content is computed with the same library calls the previous pipeline made; offsets come from a per-code-point rerun, with contextual case mappings (Greek final sigma) falling back to word-wide spans that widen but never misplace. List and map constructors cover line-number and explicit-id vocabularies. BertTokenizer, unreleased and superseded, is removed. The dl tokenizer creation builds on the encoder behind the existing Tokenizer plumbing via a package-private adapter with unchanged special-token selection, and a vocabulary missing its special tokens now fails at construction instead of at the first id mapping, pinned by a test. Parity is enforced twice: a differential suite against the reference pipeline (kept test-only as ReferenceBertPipeline) over a curated corpus plus 800 randomized inputs, and the removed class's reference token sequences ported case for case. WordpieceTokenizer is untouched. --- .../tools/tokenize/BertNormalization.java | 2 +- .../opennlp/tools/tokenize/BertTokenizer.java | 211 --------- .../tools/tokenize/WordpieceEncoder.java | 416 ++++++++++++++++++ .../src/main/java/opennlp/dl/AbstractDL.java | 40 +- .../java/opennlp/dl/EncoderTokenizer.java | 53 +++ .../dl/doccat/DocumentCategorizerDL.java | 2 +- .../opennlp/dl/namefinder/NameFinderDL.java | 2 +- .../opennlp/dl/vectors/SentenceVectorsDL.java | 2 +- .../java/opennlp/dl/CreateTokenizerTest.java | 31 +- .../tools/tokenize/ReferenceBertPipeline.java | 92 ++++ ...rdpieceEncoderReferenceSequencesTest.java} | 87 ++-- .../tools/tokenize/WordpieceEncoderTest.java | 218 +++++++++ 12 files changed, 872 insertions(+), 284 deletions(-) delete mode 100644 opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java rename opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/{BertTokenizerTest.java => WordpieceEncoderReferenceSequencesTest.java} (57%) create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java index 455f67f110..1cbfffe312 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java @@ -19,7 +19,7 @@ /** * Character classifications and text transforms of the reference BERT - * {@code BasicTokenizer}, shared by {@link BertTokenizer} and + * {@code BasicTokenizer}, shared by {@link WordpieceEncoder} and * {@link WordpieceTokenizer}. */ final class BertNormalization { diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java deleted file mode 100644 index 5517548a42..0000000000 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package opennlp.tools.tokenize; - -import java.text.Normalizer; -import java.util.Locale; -import java.util.Objects; -import java.util.Set; - -import opennlp.tools.util.Span; - -/** - * A {@link Tokenizer} implementation of the full BERT tokenization pipeline: - * basic tokenization (text normalization) followed by wordpiece tokenization. - *

- * The basic tokenization stage reproduces the reference BERT - * {@code BasicTokenizer}: - *

    - *
  1. Removal of control characters and normalization of all whitespace - * to single spaces.
    2. - *
  2. Whitespace isolation of CJK ideographs.
    3. - *
  3. For uncased models: lower casing and accent stripping - * (Unicode NFD decomposition with removal of combining marks).
    4. - *
  4. Isolation of every punctuation character as its own token.
    5. - *
- * The normalized text is then split into subwords by a - * {@link WordpieceTokenizer} sharing the same vocabulary and special tokens. - *

- * This pipeline is required for correct results with BERT-style models: - * feeding raw text directly to {@link WordpieceTokenizer} maps every token - * that does not literally appear in the vocabulary - for uncased models that - * includes every capitalized word - to the unknown token. - *

- * Whether to use the lower casing variant is a property of the model: uncased - * models (for example {@code bert-base-uncased} and the - * {@code sentence-transformers} models derived from it) require it, cased - * models must not use it. Accent stripping is coupled to lower casing, as in - * the reference implementation's default ({@code strip_accents} follows - * {@code do_lower_case} unless overridden). - *

- * For reference see: - *

- * - * @see WordpieceTokenizer - */ -public class BertTokenizer implements Tokenizer { - - /** - * Maximum characters per word before the word is replaced with the unknown - * token, matching the reference BERT implementation. - */ - private static final int MAX_WORD_CHARACTERS = 100; - - private final WordpieceTokenizer wordpieceTokenizer; - private final boolean lowerCase; - - /** - * Initializes a {@link BertTokenizer} for an uncased BERT model, - * with lower casing and accent stripping enabled. - * - * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. - */ - public BertTokenizer(Set vocabulary) { - this(vocabulary, true); - } - - /** - * Initializes a {@link BertTokenizer} with BERT special tokens. - * - * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. - * @param lowerCase {@code true} for uncased models (lower casing and accent - * stripping), {@code false} for cased models. - */ - public BertTokenizer(Set vocabulary, boolean lowerCase) { - this(vocabulary, lowerCase, WordpieceTokenizer.BERT_CLS_TOKEN, - WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN); - } - - /** - * Initializes a {@link BertTokenizer} with custom special tokens, for models - * like RoBERTa that do not use the BERT defaults. - * - * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. - * @param lowerCase {@code true} for uncased models (lower casing and - * accent stripping), {@code false} for cased models. - * @param classificationToken The CLS token. - * @param separatorToken The SEP token. - * @param unknownToken The UNK token. - */ - public BertTokenizer(Set vocabulary, boolean lowerCase, - String classificationToken, String separatorToken, String unknownToken) { - Objects.requireNonNull(vocabulary, "vocabulary must not be null"); - Objects.requireNonNull(classificationToken, "classificationToken must not be null"); - Objects.requireNonNull(separatorToken, "separatorToken must not be null"); - Objects.requireNonNull(unknownToken, "unknownToken must not be null"); - this.wordpieceTokenizer = new WordpieceTokenizer(vocabulary, - classificationToken, separatorToken, unknownToken, MAX_WORD_CHARACTERS); - this.lowerCase = lowerCase; - } - - /** - * Tokenizes the given text into wordpieces, surrounded by the classification - * and separator tokens. - * - * @param text The text to tokenize. Must not be {@code null}. - * - * @return The wordpiece tokens. - */ - @Override - public String[] tokenize(String text) { - return wordpieceTokenizer.tokenize(normalize(text)); - } - - /** - * Not supported: wordpiece tokens (subwords, {@code ##} continuations and - * special tokens) have no faithful character spans in the original text. - * - * @throws UnsupportedOperationException Always. - */ - @Override - public Span[] tokenizePos(String text) { - throw new UnsupportedOperationException( - "Wordpiece tokens cannot be mapped to character spans of the original text"); - } - - /** - * Applies the BERT basic tokenization (normalization) stage. - * - * @param text The text to normalize. Must not be {@code null}. - * - * @return The normalized text, ready for wordpiece tokenization. - */ - String normalize(String text) { - Objects.requireNonNull(text, "text must not be null"); - String normalized = cleanText(text); - normalized = isolateCjkCharacters(normalized); - if (lowerCase) { - normalized = stripAccents(normalized.toLowerCase(Locale.ROOT)); - } - return BertNormalization.isolatePunctuation(normalized); - } - - /** - * Removes invalid and control characters and normalizes all whitespace - * characters to plain spaces. - */ - private static String cleanText(String text) { - final StringBuilder cleaned = new StringBuilder(text.length()); - text.codePoints().forEach(codePoint -> { - if (codePoint == 0 || codePoint == 0xFFFD || BertNormalization.isControl(codePoint)) { - return; - } - if (BertNormalization.isWhitespace(codePoint)) { - cleaned.append(' '); - } else { - cleaned.appendCodePoint(codePoint); - } - }); - return cleaned.toString(); - } - - /** - * Surrounds every CJK ideograph with spaces, so each ideograph becomes its - * own token, matching the reference BERT treatment of Chinese text. - */ - private static String isolateCjkCharacters(String text) { - final StringBuilder spaced = new StringBuilder(text.length()); - text.codePoints().forEach(codePoint -> { - if (BertNormalization.isCjk(codePoint)) { - spaced.append(' ').appendCodePoint(codePoint).append(' '); - } else { - spaced.appendCodePoint(codePoint); - } - }); - return spaced.toString(); - } - - /** - * Removes accents by Unicode NFD decomposition followed by removal of - * combining marks ({@code Mn}). - */ - private static String stripAccents(String text) { - final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); - final StringBuilder stripped = new StringBuilder(decomposed.length()); - decomposed.codePoints().forEach(codePoint -> { - if (Character.getType(codePoint) != Character.NON_SPACING_MARK) { - stripped.appendCodePoint(codePoint); - } - }); - return stripped.toString(); - } - -} diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java new file mode 100644 index 0000000000..e74c23205d --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize; + +import java.text.Normalizer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * A {@link SubwordTokenizer} running the full BERT tokenization pipeline of the reference + * implementation: basic tokenization (control removal, whitespace normalization, CJK + * isolation, optional lower casing with accent stripping, punctuation isolation) followed by + * greedy longest-match wordpiece segmentation. + * + *

Every piece carries its vocabulary id and the span of the original text it came + * from, surviving the normalization steps that change, insert, and remove characters. The + * classification and separator pieces frame every encoding, carrying empty spans at the + * text's boundaries, so {@link #encode(CharSequence)} is never empty.

+ * + *

Ids follow the line-number convention of BERT {@code vocab.txt} files: with the list + * constructors a piece's id is its index, and with the map constructor the ids are given + * explicitly. The classification, separator, and unknown tokens must all be present in the + * vocabulary, because every emitted piece must have an id.

+ * + *

Instances are immutable and safe for concurrent use by multiple threads.

+ * + * @see WordpieceTokenizer + */ +public final class WordpieceEncoder implements SubwordTokenizer { + + // The reference implementation's limit: longer words become the unknown piece. + private static final int MAX_WORD_CHARACTERS = 100; + + private final Set vocabulary; + private final Map ids; + private final boolean lowerCase; + private final String classificationToken; + private final String separatorToken; + private final String unknownToken; + private final int classificationId; + private final int separatorId; + private final int unknownId; + + /** + * Instantiates an encoder for an uncased BERT model with the BERT special tokens. + * + * @param vocabulary The ordered vocabulary; a piece's id is its index. Must not be null, + * must not contain nulls or duplicates. + */ + public WordpieceEncoder(List vocabulary) { + this(vocabulary, true); + } + + /** + * Instantiates an encoder with the BERT special tokens. + * + * @param vocabulary The ordered vocabulary; a piece's id is its index. Must not be null, + * must not contain nulls or duplicates. + * @param lowerCase True for uncased models (lower casing and accent stripping), false for + * cased models. + */ + public WordpieceEncoder(List vocabulary, boolean lowerCase) { + this(vocabulary, lowerCase, WordpieceTokenizer.BERT_CLS_TOKEN, + WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN); + } + + /** + * Instantiates an encoder with custom special tokens, for models that do not use the BERT + * defaults. + * + * @param vocabulary The ordered vocabulary; a piece's id is its index. Must not be + * null, must not contain nulls or duplicates. + * @param lowerCase True for uncased models (lower casing and accent stripping), + * false for cased models. + * @param classificationToken The CLS token; must be in the vocabulary. + * @param separatorToken The SEP token; must be in the vocabulary. + * @param unknownToken The UNK token; must be in the vocabulary. + * @throws IllegalArgumentException Thrown if any argument is null, the vocabulary contains + * a null or duplicate entry, or a special token is missing from the vocabulary. + */ + public WordpieceEncoder(List vocabulary, boolean lowerCase, + String classificationToken, String separatorToken, + String unknownToken) { + this(byPiece(vocabulary), lowerCase, classificationToken, separatorToken, unknownToken); + } + + /** + * Instantiates an encoder from an explicit piece-to-id mapping, for vocabularies whose ids + * are not contiguous line numbers. + * + * @param vocabularyIds The piece-to-id mapping. Must not be null, must not contain + * null keys or values. + * @param lowerCase True for uncased models (lower casing and accent stripping), + * false for cased models. + * @param classificationToken The CLS token; must be in the vocabulary. + * @param separatorToken The SEP token; must be in the vocabulary. + * @param unknownToken The UNK token; must be in the vocabulary. + * @throws IllegalArgumentException Thrown if any argument is null, the mapping contains a + * null key or value, or a special token is missing from the vocabulary. + */ + public WordpieceEncoder(Map vocabularyIds, boolean lowerCase, + String classificationToken, String separatorToken, + String unknownToken) { + if (vocabularyIds == null || classificationToken == null || separatorToken == null + || unknownToken == null) { + throw new IllegalArgumentException("The vocabulary and special tokens must not be null."); + } + final Map byPiece = new HashMap<>(vocabularyIds.size() * 2); + for (final Map.Entry entry : vocabularyIds.entrySet()) { + if (entry.getKey() == null || entry.getValue() == null) { + throw new IllegalArgumentException( + "The vocabulary must not contain null pieces or ids: " + entry); + } + byPiece.put(entry.getKey(), entry.getValue()); + } + this.vocabulary = new HashSet<>(byPiece.keySet()); + this.ids = byPiece; + this.lowerCase = lowerCase; + this.classificationToken = classificationToken; + this.separatorToken = separatorToken; + this.unknownToken = unknownToken; + this.classificationId = requiredId(byPiece, classificationToken); + this.separatorId = requiredId(byPiece, separatorToken); + this.unknownId = requiredId(byPiece, unknownToken); + } + + private static Map byPiece(List vocabulary) { + if (vocabulary == null) { + throw new IllegalArgumentException("The vocabulary must not be null."); + } + final Map byPiece = new HashMap<>(vocabulary.size() * 2); + for (int id = 0; id < vocabulary.size(); id++) { + final String piece = vocabulary.get(id); + if (piece == null) { + throw new IllegalArgumentException("The vocabulary contains null at index " + id + "."); + } + if (byPiece.putIfAbsent(piece, id) != null) { + throw new IllegalArgumentException("The vocabulary contains '" + piece + + "' more than once; ids would be ambiguous."); + } + } + return byPiece; + } + + private static int requiredId(Map ids, String specialToken) { + final Integer id = ids.get(specialToken); + if (id == null) { + throw new IllegalArgumentException("The special token '" + specialToken + + "' is not in the vocabulary; every emitted piece must have an id."); + } + return id; + } + + @Override + public List encode(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("The text must not be null."); + } + final String original = text.toString(); + + // The normalized text, one original-text range per char, built through the reference + // pipeline's transformations in the reference order. + MappedText mapped = cleanAndIsolateCjk(original); + if (lowerCase) { + mapped = lowerCaseAndStripAccents(mapped); + } + mapped = isolatePunctuation(mapped); + + final List pieces = new ArrayList<>(); + pieces.add(new SubwordPiece(classificationToken, classificationId, 0, 0)); + int from = 0; + while (from < mapped.length) { + if (mapped.chars[from] == ' ') { + from++; + continue; + } + int to = from; + while (to < mapped.length && mapped.chars[to] != ' ') { + to++; + } + encodeWord(mapped, from, to, pieces); + from = to; + } + pieces.add(new SubwordPiece(separatorToken, separatorId, + original.length(), original.length())); + return pieces; + } + + // Greedy longest-match wordpiece over one whitespace-delimited word, exactly as + // WordpieceTokenizer#tokenize segments it; pieces are emitted only if the whole word is + // representable, otherwise the word becomes a single unknown piece. + private void encodeWord(MappedText mapped, int from, int to, List pieces) { + final int wordStart = mapped.starts[from]; + final int wordEnd = mapped.ends[to - 1]; + if (to - from > MAX_WORD_CHARACTERS) { + pieces.add(new SubwordPiece(unknownToken, unknownId, wordStart, wordEnd)); + return; + } + final List wordPieces = new ArrayList<>(); + int start = from; + boolean found = true; + while (start < to) { + int end = to; + found = false; + while (start < end) { + String substring = new String(mapped.chars, start, end - start); + if (start > from) { + substring = "##" + substring; + } + if (vocabulary.contains(substring)) { + wordPieces.add(new SubwordPiece(substring, ids.get(substring), + mapped.starts[start], mapped.ends[end - 1])); + start = end; + found = true; + break; + } + end--; + } + if (!found) { + break; + } + } + if (found) { + pieces.addAll(wordPieces); + } else { + pieces.add(new SubwordPiece(unknownToken, unknownId, wordStart, wordEnd)); + } + } + + // The normalized text with, for every char, the original-text range it came from. Characters + // inserted by the pipeline (isolation spaces) carry an empty range at the insertion point. + private static final class MappedText { + private char[] chars; + private int[] starts; + private int[] ends; + private int length; + + private MappedText(int capacity) { + chars = new char[capacity]; + starts = new int[capacity]; + ends = new int[capacity]; + } + + private void add(char c, int originalStart, int originalEnd) { + if (length == chars.length) { + final int capacity = Math.max(16, length * 2); + chars = Arrays.copyOf(chars, capacity); + starts = Arrays.copyOf(starts, capacity); + ends = Arrays.copyOf(ends, capacity); + } + chars[length] = c; + starts[length] = originalStart; + ends[length] = originalEnd; + length++; + } + + private void add(String s, int originalStart, int originalEnd) { + for (int i = 0; i < s.length(); i++) { + add(s.charAt(i), originalStart, originalEnd); + } + } + } + + // Text cleaning and CJK isolation in one pass: both are per-code-point + // transforms, and a CJK code point is never dropped or whitespace, so the fusion is exact. + private static MappedText cleanAndIsolateCjk(String original) { + final MappedText out = new MappedText(original.length() + 16); + int i = 0; + while (i < original.length()) { + final int codePoint = original.codePointAt(i); + final int width = Character.charCount(codePoint); + if (codePoint == 0 || codePoint == 0xFFFD || BertNormalization.isControl(codePoint)) { + i += width; + continue; + } + if (BertNormalization.isWhitespace(codePoint)) { + out.add(' ', i, i + width); + } else if (BertNormalization.isCjk(codePoint)) { + out.add(' ', i, i); + for (int c = 0; c < width; c++) { + out.add(original.charAt(i + c), i, i + width); + } + out.add(' ', i + width, i + width); + } else { + for (int c = 0; c < width; c++) { + out.add(original.charAt(i + c), i, i + width); + } + } + i += width; + } + return out; + } + + // BertNormalization#isolatePunctuation with the range of each char preserved. + private static MappedText isolatePunctuation(MappedText in) { + final MappedText out = new MappedText(in.length + 16); + int i = 0; + while (i < in.length) { + final int codePoint = codePointAt(in, i); + final int width = Character.charCount(codePoint); + if (BertNormalization.isPunctuation(codePoint)) { + out.add(' ', in.starts[i], in.starts[i]); + for (int c = 0; c < width; c++) { + out.add(in.chars[i + c], in.starts[i + c], in.ends[i + c]); + } + out.add(' ', in.ends[i + width - 1], in.ends[i + width - 1]); + } else { + for (int c = 0; c < width; c++) { + out.add(in.chars[i + c], in.starts[i + c], in.ends[i + c]); + } + } + i += width; + } + return out; + } + + // Lower casing and accent stripping with ranges preserved. The content is computed + // with whole-run library calls, applied per whitespace run + // (equivalent on whole strings: no case mapping context crosses a space, and NFD + // composition is boundary-safe at a space). The per-char ranges are reconstructed from a + // per-code-point rerun of the same transforms; if a contextual case mapping (Greek final + // sigma) makes the rerun disagree with the authoritative content, every char of that run + // falls back to the run's full range, which widens spans but never misplaces them. + private static MappedText lowerCaseAndStripAccents(MappedText in) { + final MappedText out = new MappedText(in.length + 16); + int from = 0; + while (from < in.length) { + if (in.chars[from] == ' ') { + out.add(' ', in.starts[from], in.ends[from]); + from++; + continue; + } + int to = from; + while (to < in.length && in.chars[to] != ' ') { + to++; + } + transformRun(in, from, to, out); + from = to; + } + return out; + } + + private static void transformRun(MappedText in, int from, int to, MappedText out) { + final String run = new String(in.chars, from, to - from); + final String content = stripAccents(run.toLowerCase(Locale.ROOT)); + + // Rerun per code point to learn how many output chars each input code point produces. + final StringBuilder rerun = new StringBuilder(content.length()); + final int[] produced = new int[to - from]; + int i = from; + while (i < to) { + final int codePoint = codePointAt(in, i); + final int width = Character.charCount(codePoint); + final String transformed = stripAccents( + new String(Character.toChars(codePoint)).toLowerCase(Locale.ROOT)); + rerun.append(transformed); + produced[i - from] = transformed.length(); + i += width; + } + + if (rerun.toString().equals(content)) { + int at = from; + int emitted = 0; + while (at < to) { + final int width = Character.charCount(codePointAt(in, at)); + for (int c = 0; c < produced[at - from]; c++) { + out.add(content.charAt(emitted++), in.starts[at], in.ends[at + width - 1]); + } + at += width; + } + } else { + // Contextual case mapping changed the content; the run's chars share the run's range. + out.add(content, in.starts[from], in.ends[to - 1]); + } + } + + private static String stripAccents(String text) { + final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); + final StringBuilder stripped = new StringBuilder(decomposed.length()); + decomposed.codePoints().forEach(codePoint -> { + if (Character.getType(codePoint) != Character.NON_SPACING_MARK) { + stripped.appendCodePoint(codePoint); + } + }); + return stripped.toString(); + } + + private static int codePointAt(MappedText text, int index) { + final char c = text.chars[index]; + if (Character.isHighSurrogate(c) && index + 1 < text.length + && Character.isLowSurrogate(text.chars[index + 1])) { + return Character.toCodePoint(c, text.chars[index + 1]); + } + return c; + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java index a598199270..26f00888c5 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java @@ -36,8 +36,8 @@ import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; -import opennlp.tools.tokenize.BertTokenizer; import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.WordpieceEncoder; import opennlp.tools.tokenize.WordpieceTokenizer; import opennlp.tools.util.Span; import opennlp.tools.util.normalizer.AlignedText; @@ -105,7 +105,7 @@ protected AbstractDL(final File model, final File vocabulary, final OrtSession createdSession = env.createSession(model.getPath(), sessionOptions); try { this.vocab = Map.copyOf(loadVocabFile(vocabulary)); - this.tokenizer = createBertTokenizer(vocab, lowerCase); + this.tokenizer = createPipelineTokenizer(vocab, lowerCase); } catch (IOException | RuntimeException e) { // Vocabulary/tokenizer init failed after the native session was created; close it // so a partially constructed instance never leaks the ONNX session. @@ -136,7 +136,7 @@ protected AbstractDL(final OrtEnvironment env, final OrtSession session, this.env = env; this.session = session; this.vocab = vocab; - this.tokenizer = createBertTokenizer(vocab, lowerCase); + this.tokenizer = createPipelineTokenizer(vocab, lowerCase); } /** @@ -238,38 +238,44 @@ static WordpieceTokenizer createWordpieceTokenizer( } /** - * Creates a {@link BertTokenizer} that performs the full BERT tokenization - * pipeline: basic tokenization (text normalization) followed by wordpiece. - * The special tokens are selected based on the vocabulary: if it contains - * RoBERTa-style tokens, those are used, otherwise the BERT defaults. + * Creates a {@link Tokenizer} that performs the full BERT tokenization + * pipeline: basic tokenization (text normalization) followed by wordpiece, + * backed by a {@link WordpieceEncoder}. The special tokens are selected + * based on the vocabulary: if it contains RoBERTa-style tokens, those are + * used, otherwise the BERT defaults. * * @param vocab The vocabulary map. * @param lowerCase {@code true} for uncased models (lower casing and accent * stripping), {@code false} for cased models. - * @return A configured {@link BertTokenizer}. - * @throws IllegalArgumentException Thrown if a RoBERTa-style vocabulary - * contains no supported unknown token. + * @return A configured {@link Tokenizer}. + * @throws IllegalArgumentException Thrown if the selected special tokens + * are not all present in the vocabulary. */ - protected BertTokenizer createTokenizer( + protected Tokenizer createTokenizer( final Map vocab, final boolean lowerCase) { - return createBertTokenizer(vocab, lowerCase); + return createPipelineTokenizer(vocab, lowerCase); } - static BertTokenizer createBertTokenizer( + static Tokenizer createPipelineTokenizer( final Map vocab, final boolean lowerCase) { if (vocab.containsKey( WordpieceTokenizer.ROBERTA_CLS_TOKEN) && vocab.containsKey( WordpieceTokenizer.ROBERTA_SEP_TOKEN)) { - return new BertTokenizer( - vocab.keySet(), + return new EncoderTokenizer(new WordpieceEncoder( + vocab, lowerCase, WordpieceTokenizer.ROBERTA_CLS_TOKEN, WordpieceTokenizer.ROBERTA_SEP_TOKEN, - resolveUnknownToken(vocab)); + resolveUnknownToken(vocab))); } - return new BertTokenizer(vocab.keySet(), lowerCase); + return new EncoderTokenizer(new WordpieceEncoder( + vocab, + lowerCase, + WordpieceTokenizer.BERT_CLS_TOKEN, + WordpieceTokenizer.BERT_SEP_TOKEN, + WordpieceTokenizer.BERT_UNK_TOKEN)); } /** diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java new file mode 100644 index 0000000000..43943b010f --- /dev/null +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.dl; + +import opennlp.tools.tokenize.Tokenizer; +import opennlp.tools.tokenize.WordpieceEncoder; +import opennlp.tools.util.Span; + +/** + * Adapts a {@link WordpieceEncoder} to the {@link Tokenizer} plumbing of the inference + * classes: {@link #tokenize(String)} returns the encoder's piece strings. + */ +final class EncoderTokenizer implements Tokenizer { + + private final WordpieceEncoder encoder; + + EncoderTokenizer(final WordpieceEncoder encoder) { + this.encoder = encoder; + } + + @Override + public String[] tokenize(final String text) { + return encoder.encodeToPieces(text); + } + + /** + * Not supported under the {@link Tokenizer} contract, whose spans are expected to contain + * their token's surface form; wordpiece pieces are not substrings of the input. Use + * {@link WordpieceEncoder#encode(CharSequence)} for pieces with original-text spans. + * + * @throws UnsupportedOperationException Always. + */ + @Override + public Span[] tokenizePos(final String text) { + throw new UnsupportedOperationException( + "Wordpiece tokens cannot be mapped to character spans of the original text"); + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java index e5fbe2c6a4..7b17cb3028 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/doccat/DocumentCategorizerDL.java @@ -51,7 +51,7 @@ * using ONNX models. * *

Tokenization performs BERT basic tokenization (text normalization) - * before wordpiece, see {@link opennlp.tools.tokenize.BertTokenizer}. Input + * before wordpiece, see {@link opennlp.tools.tokenize.WordpieceEncoder}. Input * text is lower cased and accent stripped by default, matching the uncased * models commonly used for classification. For cased models, set * {@link InferenceOptions#setLowerCase(boolean)} to {@code false}.

diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java index 95bb81d879..956f177d61 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/namefinder/NameFinderDL.java @@ -50,7 +50,7 @@ * An implementation of {@link opennlp.tools.namefind.TokenNameFinder} that uses ONNX models. * *

Tokenization performs BERT basic tokenization (text normalization) - * before wordpiece, see {@link opennlp.tools.tokenize.BertTokenizer}. Input + * before wordpiece, see {@link opennlp.tools.tokenize.WordpieceEncoder}. Input * text is not lower cased by default, because named entity recognition * models are commonly cased: capitalization is a strong signal for entity * boundaries. For uncased models, set diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java index 6bc76ce183..f1250ea601 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java @@ -47,7 +47,7 @@ * so the encoder attended to nothing and the output vectors were * incorrect. Additionally, tokenization now performs BERT basic * tokenization (lower casing and accent stripping by default, see - * {@link opennlp.tools.tokenize.BertTokenizer}) before wordpiece. + * {@link opennlp.tools.tokenize.WordpieceEncoder}) before wordpiece. * Output vectors change with the corrected encoding and tokenization; * any embeddings persisted from the previous behavior are not * comparable with the corrected output and must be re-embedded.

diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java index 54c4600a8e..5131ae84e7 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; -import opennlp.tools.tokenize.BertTokenizer; +import opennlp.tools.tokenize.Tokenizer; import opennlp.tools.tokenize.WordpieceTokenizer; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -52,8 +52,8 @@ private static Map robertaVocab() { } @Test - void testCreatesLowerCasingBertTokenizer() { - final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(bertVocab(), true); + void testCreatesLowerCasingPipelineTokenizer() { + final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(bertVocab(), true); // Capitalized input must be lower cased before the wordpiece lookup. assertArrayEquals(new String[] { @@ -62,8 +62,8 @@ void testCreatesLowerCasingBertTokenizer() { } @Test - void testCreatesCasePreservingBertTokenizer() { - final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(bertVocab(), false); + void testCreatesCasePreservingPipelineTokenizer() { + final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(bertVocab(), false); // Without lower casing, capitalized words miss the lowercase-only vocabulary. assertArrayEquals(new String[] { @@ -74,7 +74,7 @@ void testCreatesCasePreservingBertTokenizer() { @Test void testSelectsRobertaSpecialTokens() { - final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(robertaVocab(), false); + final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(robertaVocab(), false); assertArrayEquals(new String[] { WordpieceTokenizer.ROBERTA_CLS_TOKEN, "hello", WordpieceTokenizer.ROBERTA_UNK_TOKEN, @@ -88,7 +88,7 @@ void testFallsBackToBertUnknownToken() { vocab.remove(WordpieceTokenizer.ROBERTA_UNK_TOKEN); vocab.put(WordpieceTokenizer.BERT_UNK_TOKEN, 2); - final BertTokenizer tokenizer = AbstractDL.createBertTokenizer(vocab, false); + final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(vocab, false); assertArrayEquals(new String[] { WordpieceTokenizer.ROBERTA_CLS_TOKEN, "hello", WordpieceTokenizer.BERT_UNK_TOKEN, @@ -101,10 +101,25 @@ void testRejectsRobertaVocabularyWithoutUnknownToken() { final Map vocab = robertaVocab(); vocab.remove(WordpieceTokenizer.ROBERTA_UNK_TOKEN); - assertThrows(IllegalArgumentException.class, () -> AbstractDL.createBertTokenizer(vocab, false)); + assertThrows(IllegalArgumentException.class, () -> AbstractDL.createPipelineTokenizer(vocab, false)); assertThrows(IllegalArgumentException.class, () -> AbstractDL.createWordpieceTokenizer(vocab)); } + @Test + void testTokenizePosIsUnsupported() { + final Tokenizer tokenizer = AbstractDL.createPipelineTokenizer(bertVocab(), true); + assertThrows(UnsupportedOperationException.class, () -> tokenizer.tokenizePos("the fox")); + } + + @Test + void testRejectsBertVocabularyMissingSpecialTokensAtCreation() { + final Map vocab = bertVocab(); + vocab.remove(WordpieceTokenizer.BERT_UNK_TOKEN); + + assertThrows(IllegalArgumentException.class, + () -> AbstractDL.createPipelineTokenizer(vocab, true)); + } + @Test void testResolveLowerCaseUsesComponentDefaultWhenUnset() { final InferenceOptions options = new InferenceOptions(); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java new file mode 100644 index 0000000000..867c28dbfa --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize; + +import java.text.Normalizer; +import java.util.Locale; +import java.util.Set; + +/** + * The reference BERT basic-tokenization stage feeding {@link WordpieceTokenizer}, kept + * test-only as the frozen differential baseline for {@link WordpieceEncoderTest}: the + * encoder's piece sequence must match this pipeline exactly. + */ +final class ReferenceBertPipeline { + + private static final int MAX_WORD_CHARACTERS = 100; + + private final WordpieceTokenizer wordpieceTokenizer; + private final boolean lowerCase; + + ReferenceBertPipeline(Set vocabulary, boolean lowerCase) { + this.wordpieceTokenizer = new WordpieceTokenizer(vocabulary, + WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, + WordpieceTokenizer.BERT_UNK_TOKEN, MAX_WORD_CHARACTERS); + this.lowerCase = lowerCase; + } + + String[] tokenize(String text) { + return wordpieceTokenizer.tokenize(normalize(text)); + } + + private String normalize(String text) { + String normalized = cleanText(text); + normalized = isolateCjkCharacters(normalized); + if (lowerCase) { + normalized = stripAccents(normalized.toLowerCase(Locale.ROOT)); + } + return BertNormalization.isolatePunctuation(normalized); + } + + private static String cleanText(String text) { + final StringBuilder cleaned = new StringBuilder(text.length()); + text.codePoints().forEach(codePoint -> { + if (codePoint == 0 || codePoint == 0xFFFD || BertNormalization.isControl(codePoint)) { + return; + } + if (BertNormalization.isWhitespace(codePoint)) { + cleaned.append(' '); + } else { + cleaned.appendCodePoint(codePoint); + } + }); + return cleaned.toString(); + } + + private static String isolateCjkCharacters(String text) { + final StringBuilder spaced = new StringBuilder(text.length()); + text.codePoints().forEach(codePoint -> { + if (BertNormalization.isCjk(codePoint)) { + spaced.append(' ').appendCodePoint(codePoint).append(' '); + } else { + spaced.appendCodePoint(codePoint); + } + }); + return spaced.toString(); + } + + private static String stripAccents(String text) { + final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); + final StringBuilder stripped = new StringBuilder(decomposed.length()); + decomposed.codePoints().forEach(codePoint -> { + if (Character.getType(codePoint) != Character.NON_SPACING_MARK) { + stripped.appendCodePoint(codePoint); + } + }); + return stripped.toString(); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java similarity index 57% rename from opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java rename to opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java index d8f706f4ef..68e8f219f7 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java @@ -14,36 +14,40 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package opennlp.tools.tokenize; -import java.util.Set; +import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; /** - * Tests {@link BertTokenizer}. + * The reference token sequences of the removed full-pipeline {@code Tokenizer}, re-asserted + * against {@link WordpieceEncoder}. *

* All expected token sequences in this test were generated with the HuggingFace * {@code tokenizers} reference implementation ({@code BertWordPieceTokenizer}) * using the same vocabulary, so they are verified to be identical to the - * reference BERT tokenization. + * reference BERT tokenization. The encoder requires its special tokens to be + * present in the vocabulary (every piece must have an id), so the vocabularies + * here include them; the token sequences are unchanged. */ -public class BertTokenizerTest { +public class WordpieceEncoderReferenceSequencesTest { - private static final Set VOCABULARY = Set.of( + private static final List VOCABULARY = List.of( + "[CLS]", "[SEP]", "[UNK]", "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog", "em", "##bed", "##ding", "##s", "wurttemberg", "strasse", "grosse", "don", "t", "wait", "what", ".", ",", "?", "!", "'", - "\u6211", "\u7231", // CJK: 我 爱 + "\u6211", "\u7231", // CJK "natural", "language", "processing"); @Test void testLowerCasesCapitalizedWords() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); - final String[] tokens = tokenizer.tokenize("The quick brown fox jumps over the lazy dog."); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); + final String[] tokens = + encoder.encodeToPieces("The quick brown fox jumps over the lazy dog."); final String[] expected = {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog", ".", "[SEP]"}; @@ -52,8 +56,8 @@ void testLowerCasesCapitalizedWords() { @Test void testLowerCasesBeforeWordpieceSplitting() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); - final String[] tokens = tokenizer.tokenize("Embeddings"); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); + final String[] tokens = encoder.encodeToPieces("Embeddings"); final String[] expected = {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"}; Assertions.assertArrayEquals(expected, tokens); @@ -61,10 +65,10 @@ void testLowerCasesBeforeWordpieceSplitting() { @Test void testStripsAccentsButKeepsNonCombiningCharacters() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); - // ü decomposes to u + combining diaeresis and the mark is stripped; - // ß is not a combining mark and must survive, leaving an OOV token. - final String[] tokens = tokenizer.tokenize("W\u00fcrttemberg Stra\u00dfe"); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); + // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped; + // the sharp s is not a combining mark and must survive, leaving an OOV token. + final String[] tokens = encoder.encodeToPieces("W\u00fcrttemberg Stra\u00dfe"); final String[] expected = {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"}; Assertions.assertArrayEquals(expected, tokens); @@ -72,8 +76,8 @@ void testStripsAccentsButKeepsNonCombiningCharacters() { @Test void testSplitsPunctuationRunsIntoSingleCharacters() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); - final String[] tokens = tokenizer.tokenize("Wait... what?!"); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); + final String[] tokens = encoder.encodeToPieces("Wait... what?!"); final String[] expected = {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"}; Assertions.assertArrayEquals(expected, tokens); @@ -81,8 +85,8 @@ void testSplitsPunctuationRunsIntoSingleCharacters() { @Test void testSplitsApostrophesAsPunctuation() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); - final String[] tokens = tokenizer.tokenize("don't"); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); + final String[] tokens = encoder.encodeToPieces("don't"); final String[] expected = {"[CLS]", "don", "'", "t", "[SEP]"}; Assertions.assertArrayEquals(expected, tokens); @@ -90,8 +94,8 @@ void testSplitsApostrophesAsPunctuation() { @Test void testIsolatesCjkIdeographs() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); - final String[] tokens = tokenizer.tokenize("\u6211\u7231natural language processing"); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); + final String[] tokens = encoder.encodeToPieces("\u6211\u7231natural language processing"); final String[] expected = {"[CLS]", "\u6211", "\u7231", "natural", "language", "processing", "[SEP]"}; @@ -100,10 +104,10 @@ void testIsolatesCjkIdeographs() { @Test void testCleansControlCharactersAndNormalizesWhitespace() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); // Tab and no-break space are whitespace; the NUL character is removed, // joining "brown" and "fox" into one out-of-vocabulary token. - final String[] tokens = tokenizer.tokenize("the\tquick\u00a0brown\u0000fox"); + final String[] tokens = encoder.encodeToPieces("the\tquick\u00a0brown\u0000fox"); final String[] expected = {"[CLS]", "the", "quick", "[UNK]", "[SEP]"}; Assertions.assertArrayEquals(expected, tokens); @@ -111,11 +115,11 @@ void testCleansControlCharactersAndNormalizesWhitespace() { @Test void testRemovesPrivateUseAndUnassignedCharacters() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); // The reference implementation treats all C* categories as control // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn) // are removed, joining the surrounding text into one OOV token. - final String[] tokens = tokenizer.tokenize("fox\ue000jumps and fox\ufdd0jumps"); + final String[] tokens = encoder.encodeToPieces("fox\ue000jumps and fox\ufdd0jumps"); final String[] expected = {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"}; Assertions.assertArrayEquals(expected, tokens); @@ -123,19 +127,21 @@ void testRemovesPrivateUseAndUnassignedCharacters() { @Test void testRejectsNullSpecialTokens() { - Assertions.assertThrows(NullPointerException.class, - () -> new BertTokenizer(VOCABULARY, true, null, "[SEP]", "[UNK]")); - Assertions.assertThrows(NullPointerException.class, - () -> new BertTokenizer(VOCABULARY, true, "[CLS]", null, "[UNK]")); - Assertions.assertThrows(NullPointerException.class, - () -> new BertTokenizer(VOCABULARY, true, "[CLS]", "[SEP]", null)); + // The encoder's contract throws IllegalArgumentException where the removed class threw + // NullPointerException. + Assertions.assertThrows(IllegalArgumentException.class, + () -> new WordpieceEncoder(VOCABULARY, true, null, "[SEP]", "[UNK]")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new WordpieceEncoder(VOCABULARY, true, "[CLS]", null, "[UNK]")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new WordpieceEncoder(VOCABULARY, true, "[CLS]", "[SEP]", null)); } @Test void testCasedModeKeepsCaseAndAccents() { - final Tokenizer tokenizer = new BertTokenizer( - Set.of("The", "W\u00fcrttemberg", "fox"), false); - final String[] tokens = tokenizer.tokenize("The W\u00fcrttemberg fox"); + final WordpieceEncoder encoder = new WordpieceEncoder( + List.of("[CLS]", "[SEP]", "[UNK]", "The", "W\u00fcrttemberg", "fox"), false); + final String[] tokens = encoder.encodeToPieces("The W\u00fcrttemberg fox"); final String[] expected = {"[CLS]", "The", "W\u00fcrttemberg", "fox", "[SEP]"}; Assertions.assertArrayEquals(expected, tokens); @@ -143,20 +149,13 @@ void testCasedModeKeepsCaseAndAccents() { @Test void testCustomSpecialTokens() { - final Tokenizer tokenizer = new BertTokenizer(Set.of("the", "fox"), true, + final WordpieceEncoder encoder = new WordpieceEncoder( + List.of("", "", "", "the", "fox"), true, WordpieceTokenizer.ROBERTA_CLS_TOKEN, WordpieceTokenizer.ROBERTA_SEP_TOKEN, WordpieceTokenizer.ROBERTA_UNK_TOKEN); - final String[] tokens = tokenizer.tokenize("The unknown fox"); + final String[] tokens = encoder.encodeToPieces("The unknown fox"); final String[] expected = {"", "the", "", "fox", ""}; Assertions.assertArrayEquals(expected, tokens); } - - @Test - void testTokenizePosIsUnsupported() { - final Tokenizer tokenizer = new BertTokenizer(VOCABULARY); - Assertions.assertThrows(UnsupportedOperationException.class, - () -> tokenizer.tokenizePos("the fox")); - } - } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java new file mode 100644 index 0000000000..610bba5ed5 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The encoder held against {@link ReferenceBertPipeline} for piece-sequence parity (the encoder's + * contract is "the same pipeline, plus ids and spans"), plus exact hand-computed span + * assertions through every normalization step that changes, inserts, or removes characters. + */ +class WordpieceEncoderTest { + + // Ids are indices: [PAD]=0, [UNK]=1, [CLS]=2, [SEP]=3, hello=4, world=5, ##s=6, won=7, + // ##der=8, ##ful=9, ca=10, ##fe=11, istanbul=12, U+4E2D=13, U+56FD=14, .=15, ,=16, !=17, + // he=18, ##llo=19, Greek "sofos" with a final sigma=20. + private static final List VOCAB = List.of( + "[PAD]", "[UNK]", "[CLS]", "[SEP]", "hello", "world", "##s", "won", "##der", "##ful", + "ca", "##fe", "istanbul", "\u4E2D", "\u56FD", ".", ",", "!", "he", "##llo", + "\u03C3\u03BF\u03C6\u03BF\u03C2"); + + private static WordpieceEncoder uncased() { + return new WordpieceEncoder(VOCAB); + } + + private static void assertPiece(SubwordPiece piece, String expectedPiece, int expectedId, + int expectedStart, int expectedEnd) { + assertEquals(expectedPiece, piece.piece()); + assertEquals(expectedId, piece.id()); + assertEquals(expectedStart, piece.start(), "start of " + piece); + assertEquals(expectedEnd, piece.end(), "end of " + piece); + } + + @Test + void testPieceSequenceMatchesTheReferencePipelineOnCuratedInputs() { + final ReferenceBertPipeline reference = new ReferenceBertPipeline(new HashSet<>(VOCAB), true); + final WordpieceEncoder encoder = uncased(); + final String[] inputs = { + "", + " ", + "Hello, WORLD!", + "Wonderful", + "hellos", + // An accented e, stripped by NFD decomposition. + "Caf\u00E9", + // The Turkish dotted capital I: lower cases to two chars, then the dot strips away. + "\u0130stanbul", + // CJK ideographs are isolated into single-character tokens. + "\u4E2D\u56FD is CJK", + // Greek upper case: the trailing sigma takes the contextual final-sigma mapping. + "\u03A3\u039F\u03A6\u039F\u03A3", + // The NBSP is whitespace in the BERT sense. + "hello\u00A0world", + // NUL and the zero-width space are removed by the cleaning stage. + "a\u0000b\u200Bc", + // An emoji: unknown to the vocabulary, and a surrogate pair. + "\uD83D\uDE00", + "!!!", + "a".repeat(101), + "he said: \u00ABhello\u00BB.", + }; + for (final String input : inputs) { + assertArrayEquals(reference.tokenize(input), encoder.encodeToPieces(input), + "parity broke on: " + input); + } + } + + @Test + void testPieceSequenceMatchesTheReferencePipelineOnRandomInputs() { + final int[] pool = {'a', 'b', 'A', 'B', 'z', ' ', ' ', '\t', 0x00A0, '.', '!', ',', + 0x0301, 0x00E9, 0x0130, 0x03A3, 0x03C3, 0x03BF, 0x4E2D, 0xFFFD, 0x200B, 0x1F600, 0}; + final Random random = new Random(42); + for (final boolean lowerCase : new boolean[] {true, false}) { + final ReferenceBertPipeline reference = + new ReferenceBertPipeline(new HashSet<>(VOCAB), lowerCase); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCAB, lowerCase); + for (int round = 0; round < 400; round++) { + final StringBuilder text = new StringBuilder(); + final int length = random.nextInt(25); + for (int i = 0; i < length; i++) { + text.appendCodePoint(pool[random.nextInt(pool.length)]); + } + final String input = text.toString(); + assertArrayEquals(reference.tokenize(input), encoder.encodeToPieces(input), + "parity broke on: " + input); + + // Span invariants: within bounds and never moving backwards. + int previousStart = 0; + for (final SubwordPiece piece : encoder.encode(input)) { + assertTrue(piece.start() >= previousStart && piece.end() <= input.length(), + "span out of order or bounds in " + input + ": " + piece); + previousStart = piece.start(); + } + } + } + } + + @Test + void testSpansSurvivePunctuationIsolationAndCaseFolding() { + final List pieces = uncased().encode("Hello, WORLD!"); + assertEquals(6, pieces.size()); + assertPiece(pieces.get(0), "[CLS]", 2, 0, 0); + assertPiece(pieces.get(1), "hello", 4, 0, 5); + assertPiece(pieces.get(2), ",", 16, 5, 6); + assertPiece(pieces.get(3), "world", 5, 7, 12); + assertPiece(pieces.get(4), "!", 17, 12, 13); + assertPiece(pieces.get(5), "[SEP]", 3, 13, 13); + } + + @Test + void testSpansSurviveAccentStripping() { + // The accent is stripped by NFD, yet ##fe still covers the accented surface. + final List pieces = uncased().encode("Caf\u00E9"); + assertEquals(4, pieces.size()); + assertPiece(pieces.get(1), "ca", 10, 0, 2); + assertPiece(pieces.get(2), "##fe", 11, 2, 4); + } + + @Test + void testSpansSurviveLengthChangingLowerCasing() { + // The Turkish dotted capital I lower cases to two chars before the combining dot strips; + // the piece still covers the original eight chars. + final List pieces = uncased().encode("\u0130stanbul"); + assertEquals(3, pieces.size()); + assertPiece(pieces.get(1), "istanbul", 12, 0, 8); + } + + @Test + void testCjkIsolationYieldsOnePieceAndSpanPerIdeograph() { + final List pieces = uncased().encode("\u4E2D\u56FD"); + assertEquals(4, pieces.size()); + assertPiece(pieces.get(1), "\u4E2D", 13, 0, 1); + assertPiece(pieces.get(2), "\u56FD", 14, 1, 2); + } + + @Test + void testUnknownWordCoversItsWholeSurfaceIncludingRemovedChars() { + // NUL and the zero-width space are removed by cleaning, so one word "abc" remains; it is + // not representable and becomes the unknown piece spanning the full original surface. + final List pieces = uncased().encode("a\u0000b\u200Bc"); + assertEquals(3, pieces.size()); + assertPiece(pieces.get(1), "[UNK]", 1, 0, 5); + } + + @Test + void testContextualCaseMappingFallsBackToWordWideSpans() { + // Greek final sigma is a contextual mapping the per-char rerun cannot reproduce, so the + // word's pieces fall back to spanning the whole word; content parity is asserted in the + // differential tests above. + final List pieces = + uncased().encode("\u03A3\u039F\u03A6\u039F\u03A3"); + assertEquals(3, pieces.size()); + assertPiece(pieces.get(1), + "\u03C3\u03BF\u03C6\u03BF\u03C2", 20, 0, 5); + } + + @Test + void testEncodeToIdsCarriesVocabularyLineNumbers() { + assertArrayEquals(new int[] {2, 4, 5, 6, 3}, uncased().encodeToIds("Hello worldS")); + } + + @Test + void testCasedEncoderKeepsCase() { + final List vocabulary = new ArrayList<>(VOCAB); + vocabulary.add("Hello"); + final WordpieceEncoder cased = new WordpieceEncoder(vocabulary, false); + final List pieces = cased.encode("Hello hello"); + assertPiece(pieces.get(1), "Hello", vocabulary.size() - 1, 0, 5); + assertPiece(pieces.get(2), "hello", 4, 6, 11); + } + + @Test + void testEmptyAndBlankTextEncodeToTheFramePiecesOnly() { + for (final String input : new String[] {"", " "}) { + final List pieces = uncased().encode(input); + assertEquals(2, pieces.size()); + assertPiece(pieces.get(0), "[CLS]", 2, 0, 0); + assertPiece(pieces.get(1), "[SEP]", 3, input.length(), input.length()); + } + } + + @Test + void testValidationFailsLoudly() { + assertThrows(IllegalArgumentException.class, () -> new WordpieceEncoder(null)); + assertThrows(IllegalArgumentException.class, + () -> new WordpieceEncoder(List.of("[CLS]", "[SEP]"))); + assertThrows(IllegalArgumentException.class, + () -> new WordpieceEncoder(List.of("[CLS]", "[SEP]", "[UNK]", "dup", "dup"))); + final List withNull = new ArrayList<>(VOCAB); + withNull.add(null); + assertThrows(IllegalArgumentException.class, () -> new WordpieceEncoder(withNull)); + assertThrows(IllegalArgumentException.class, () -> uncased().encode(null)); + } +} From abbf3e6bc94b1806acf8777f84b9d092b34a15f5 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 15:15:22 -0400 Subject: [PATCH 05/82] OPENNLP-1885: Document the hand-rolled protobuf reader rationale and refactor trigger --- .../opennlp/subword/sentencepiece/ModelProtoReader.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java index 4e75f49c97..58f9dd83e1 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java @@ -28,6 +28,14 @@ * directly and keeps only the fields inference needs: the pieces with scores and types, the * normalizer spec, the handful of trainer-spec fields that change runtime behavior, and the * embedded self-test samples. Unknown fields are skipped, malformed input fails loudly.

+ * + *

This is a self-contained wire reader, not a performance optimization: model loading happens + * once and is not on any hot path. It exists so this module reads a protobuf-encoded file without + * adding a {@code protobuf-java} runtime dependency, which the project does not otherwise use, for + * a load-time parse of a schema that is stable in practice. If OpenNLP ever takes on a real + * protobuf dependency for other reasons, this hand-rolled reader should be retired in favor of it: + * generate from {@code sentencepiece_model.proto} (or parse via the descriptor API) and delete + * this class, since the dependency-avoidance rationale no longer holds.

*/ final class ModelProtoReader { From 5c78b97f3864054dfb768958958ec62c42533353 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 15:29:54 -0400 Subject: [PATCH 06/82] OPENNLP-1885: Trim commentary and tighten javadoc per review conventions --- .../tools/tokenize/WordpieceEncoder.java | 50 +++++++++++++------ .../src/main/java/opennlp/dl/AbstractDL.java | 10 ++++ .../java/opennlp/dl/EncoderTokenizer.java | 6 +++ .../subword/sentencepiece/BpeEncoder.java | 28 +++++++---- .../subword/sentencepiece/ByteBuilder.java | 30 +++++++++++ .../sentencepiece/DoubleArrayTrie.java | 7 ++- .../subword/sentencepiece/IntBuilder.java | 24 +++++++++ .../SentencePieceNormalizer.java | 48 ++++++++++++------ .../sentencepiece/SentencePieceTokenizer.java | 3 +- .../subword/sentencepiece/UnigramEncoder.java | 9 +--- .../subword/sentencepiece/Utf8Text.java | 9 ++-- 11 files changed, 168 insertions(+), 56 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java index e74c23205d..65c73dab0c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java @@ -206,9 +206,15 @@ public List encode(CharSequence text) { return pieces; } - // Greedy longest-match wordpiece over one whitespace-delimited word, exactly as - // WordpieceTokenizer#tokenize segments it; pieces are emitted only if the whole word is - // representable, otherwise the word becomes a single unknown piece. + /** + * Greedily longest-match segments one whitespace-delimited word; the pieces are emitted only if + * the whole word is representable, otherwise the word becomes a single unknown piece. + * + * @param mapped The normalized text with per-character original-text ranges. + * @param from The inclusive start of the word in {@code mapped}. + * @param to The exclusive end of the word in {@code mapped}. + * @param pieces The output list to append to. + */ private void encodeWord(MappedText mapped, int from, int to, List pieces) { final int wordStart = mapped.starts[from]; final int wordEnd = mapped.ends[to - 1]; @@ -247,8 +253,10 @@ private void encodeWord(MappedText mapped, int from, int to, List } } - // The normalized text with, for every char, the original-text range it came from. Characters - // inserted by the pipeline (isolation spaces) carry an empty range at the insertion point. + /** + * The normalized text with, for every char, the original-text range it came from. Characters + * inserted by the pipeline (isolation spaces) carry an empty range at the insertion point. + */ private static final class MappedText { private char[] chars; private int[] starts; @@ -281,8 +289,13 @@ private void add(String s, int originalStart, int originalEnd) { } } - // Text cleaning and CJK isolation in one pass: both are per-code-point - // transforms, and a CJK code point is never dropped or whitespace, so the fusion is exact. + /** + * Cleans the text (control and whitespace normalization) and isolates CJK code points in one + * pass, recording the original-text range of every output character. + * + * @param original The original input text. + * @return The cleaned, CJK-isolated text with per-character ranges. + */ private static MappedText cleanAndIsolateCjk(String original) { final MappedText out = new MappedText(original.length() + 16); int i = 0; @@ -311,7 +324,13 @@ private static MappedText cleanAndIsolateCjk(String original) { return out; } - // BertNormalization#isolatePunctuation with the range of each char preserved. + /** + * Isolates punctuation, surrounding each punctuation code point with spaces, preserving the + * original-text range of every character. + * + * @param in The input text with per-character ranges. + * @return The punctuation-isolated text with per-character ranges. + */ private static MappedText isolatePunctuation(MappedText in) { final MappedText out = new MappedText(in.length + 16); int i = 0; @@ -334,13 +353,14 @@ private static MappedText isolatePunctuation(MappedText in) { return out; } - // Lower casing and accent stripping with ranges preserved. The content is computed - // with whole-run library calls, applied per whitespace run - // (equivalent on whole strings: no case mapping context crosses a space, and NFD - // composition is boundary-safe at a space). The per-char ranges are reconstructed from a - // per-code-point rerun of the same transforms; if a contextual case mapping (Greek final - // sigma) makes the rerun disagree with the authoritative content, every char of that run - // falls back to the run's full range, which widens spans but never misplaces them. + /** + * Lower cases and strips accents, preserving the original-text range of every character. When a + * contextual case mapping prevents a per-character range from being recovered, the whole + * whitespace run falls back to its full range, which widens spans but never misplaces them. + * + * @param in The input text with per-character ranges. + * @return The lower-cased, accent-stripped text with per-character ranges. + */ private static MappedText lowerCaseAndStripAccents(MappedText in) { final MappedText out = new MappedText(in.length + 16); int from = 0; diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java index 26f00888c5..01e4d68ca4 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java @@ -257,6 +257,16 @@ protected Tokenizer createTokenizer( return createPipelineTokenizer(vocab, lowerCase); } + /** + * Builds the pipeline tokenizer, selecting the RoBERTa special tokens when the vocabulary + * carries them and the BERT defaults otherwise. + * + * @param vocab The vocabulary map. + * @param lowerCase {@code true} for uncased models, {@code false} for cased models. + * @return A configured {@link Tokenizer}. + * @throws IllegalArgumentException Thrown if the selected special tokens are not all present in + * the vocabulary. + */ static Tokenizer createPipelineTokenizer( final Map vocab, final boolean lowerCase) { if (vocab.containsKey( diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java index 43943b010f..5422c6773d 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java @@ -29,10 +29,16 @@ final class EncoderTokenizer implements Tokenizer { private final WordpieceEncoder encoder; + /** + * Instantiates the adapter. + * + * @param encoder The encoder whose pieces this tokenizer returns. + */ EncoderTokenizer(final WordpieceEncoder encoder) { this.encoder = encoder; } + /** {@inheritDoc} */ @Override public String[] tokenize(final String text) { return encoder.encodeToPieces(text); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java index e7430032ae..c6ad75542c 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java @@ -28,11 +28,8 @@ * user-defined symbols, which are frozen whole) and adjacent pairs merge greedily, highest piece * score first, until no adjacent pair forms a vocabulary piece. * - *

This ports the reference implementation's agenda algorithm: candidate pairs sit in a - * priority queue ordered by score with ties broken towards the leftmost pair, stale entries are - * detected by a length check when popped, and merges that land on a piece marked unused are - * re-segmented back into their constituents afterwards. Only pieces of the normal, user-defined, - * and unused types participate in merges.

+ *

Only pieces of the normal, user-defined, and unused types participate in merges; a merge that + * lands on an unused piece is re-segmented back into its constituents.

*/ final class BpeEncoder { @@ -67,8 +64,10 @@ final class BpeEncoder { this.userDefinedMatcher = userDefinedMatcher; } - // A candidate merge of the symbols at indices left and right; size is the merged byte length - // used to detect staleness after either side has changed. + /** + * A candidate merge of the symbols at indices {@code left} and {@code right}; {@code size} is the + * merged byte length, used to detect staleness after either side has changed. + */ private record Pair(int left, int right, float score, int size) { } @@ -179,9 +178,18 @@ private void maybeAddPair(byte[] normalized, int[] from, int[] to, boolean[] fre } } - // Emits a symbol, splitting a piece of the unused type back into the pieces it was merged - // from. Positions are assigned by a running cursor; constituent byte lengths always sum to the - // merged length, so the cursor stays aligned with the normalized bytes. + /** + * Emits a symbol, splitting a piece of the unused type back into the pieces it was merged from. + * Positions are assigned by a running cursor; constituent byte lengths always sum to the merged + * length, so the cursor stays aligned with the normalized bytes. + * + * @param piece The piece content to emit. + * @param consumed The running byte cursor into the normalized text. + * @param depth The current recursion depth. + * @param revMerge The map from a merged piece to its two constituents. + * @param output The segment list to append to. + * @return The updated byte cursor. + */ private int resegment(String piece, int consumed, int depth, Map revMerge, List output) { final Integer mapped = pieces.get(piece); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java index 1d2c3ab952..95a02fddd5 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java @@ -24,10 +24,20 @@ final class ByteBuilder { private byte[] data; private int length; + /** + * Instantiates the buffer. + * + * @param capacity The initial capacity hint. + */ ByteBuilder(int capacity) { data = new byte[Math.max(capacity, 16)]; } + /** + * Appends one byte. + * + * @param b The byte to append. + */ void append(byte b) { if (length == data.length) { data = Arrays.copyOf(data, data.length + (data.length >> 1)); @@ -35,6 +45,13 @@ void append(byte b) { data[length++] = b; } + /** + * Appends a run of bytes. + * + * @param source The source array. + * @param from The inclusive start offset in {@code source}. + * @param count The number of bytes to append. + */ void append(byte[] source, int from, int count) { while (length + count > data.length) { data = Arrays.copyOf(data, data.length + (data.length >> 1)); @@ -43,14 +60,26 @@ void append(byte[] source, int from, int count) { length += count; } + /** {@return the number of valid bytes} */ int length() { return length; } + /** + * Shrinks the valid length. + * + * @param newLength The new length, not greater than the current length. + */ void truncate(int newLength) { length = newLength; } + /** + * Tests whether the valid bytes end with the given suffix. + * + * @param suffix The suffix to test. + * @return {@code true} when the buffer ends with {@code suffix}. + */ boolean endsWith(byte[] suffix) { if (length < suffix.length) { return false; @@ -58,6 +87,7 @@ boolean endsWith(byte[] suffix) { return Arrays.equals(data, length - suffix.length, length, suffix, 0, suffix.length); } + /** {@return a trimmed copy of the valid bytes} */ byte[] toArray() { return Arrays.copyOf(data, length); } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java index c88fbe161b..967b525fe5 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java @@ -104,7 +104,12 @@ boolean hasTransitionFromRoot(int b) { return (units[nodePos] & 0x800000FF) == b; } - // The offset from a unit to its children, as encoded by Darts-clone. + /** + * Returns the offset from a unit to its children, as encoded by Darts-clone. + * + * @param unit The unit word. + * @return The child offset. + */ private static int offset(int unit) { return (unit >>> 10) << ((unit & (1 << 9)) >>> 6); } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java index ec585685c1..25dad2ec92 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java @@ -24,10 +24,20 @@ final class IntBuilder { private int[] data; private int length; + /** + * Instantiates the buffer. + * + * @param capacity The initial capacity hint. + */ IntBuilder(int capacity) { data = new int[Math.max(capacity, 16)]; } + /** + * Appends one value. + * + * @param value The value to append. + */ void append(int value) { if (length == data.length) { data = Arrays.copyOf(data, data.length + (data.length >> 1)); @@ -35,6 +45,13 @@ void append(int value) { data[length++] = value; } + /** + * Reads a value by index. + * + * @param index An index in {@code [0, length())}. + * @return The value at {@code index}. + * @throws IndexOutOfBoundsException Thrown if {@code index} is out of range. + */ int get(int index) { if (index >= length) { throw new IndexOutOfBoundsException("index " + index + " is outside [0, " + length + ")"); @@ -42,14 +59,21 @@ int get(int index) { return data[index]; } + /** {@return the number of valid values} */ int length() { return length; } + /** + * Shrinks the valid length. + * + * @param newLength The new length, not greater than the current length. + */ void truncate(int newLength) { length = newLength; } + /** {@return a trimmed copy of the valid values} */ int[] toArray() { return Arrays.copyOf(data, length); } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java index d99d4b33ce..8c6ed5a4d7 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -25,9 +25,6 @@ * {@code normToOrig}, mapping every normalized byte to the offset of the original byte chunk it * was derived from, with one trailing entry for the end position; that map is what lets every * downstream piece report an exact span of the caller's text.

- * - *

This mirrors the reference implementation's normalizer semantics rule for rule, since parity - * of both the normalized bytes and the offset map is what the tests assert.

*/ final class SentencePieceNormalizer { @@ -118,10 +115,11 @@ final class SentencePieceNormalizer { record Normalized(byte[] bytes, int length, int[] normToOrig) { } - // One normalization step: `consumed` input bytes produced `data[from, to)`. The data array is - // the input itself (pass-through), the replacement blob, or the replacement character. One - // mutable scratch per normalize call, refilled per chunk, so the scan allocates nothing per - // code point. + /** + * One normalization step: {@code consumed} input bytes produced {@code data[from, to)}. The data + * array is the input itself (pass-through), the replacement blob, or the replacement character. + * A single mutable scratch is refilled per chunk so the scan allocates nothing per code point. + */ private static final class Chunk { private byte[] data; @@ -249,10 +247,17 @@ private static void appendSpace(ByteBuilder normalized, IntBuilder normToOrig, } } - // Fills the scratch with the normalized form of the longest applicable prefix of - // input[from, inputLength): a user-defined symbol passes through raw, otherwise the longest - // character-map rule applies, otherwise one code point passes through raw (or becomes U+FFFD - // when the lead byte is malformed). + /** + * Fills the scratch with the normalized form of the longest applicable prefix of + * {@code input[from, inputLength)}: a user-defined symbol passes through raw, otherwise the + * longest character-map rule applies, otherwise one code point passes through raw (or becomes + * U+FFFD when the lead byte is malformed). + * + * @param input The UTF-8 input buffer. + * @param inputLength The number of valid bytes in {@code input}. + * @param from The offset to normalize from. + * @param chunk The scratch to fill. + */ private void normalizePrefix(byte[] input, int inputLength, int from, Chunk chunk) { if (userDefinedMatcher != null) { final int matched = longestUserDefinedMatch(input, inputLength, from); @@ -314,8 +319,13 @@ private int longestUserDefinedMatch(byte[] input, int inputLength, int from) { return longest; } - // The byte length of a UTF-8 sequence by its lead byte, as the reference implementation - // computes it: trail and malformed lead bytes report one byte. + /** + * Returns the byte length of a UTF-8 sequence from its lead byte; trail and malformed lead bytes + * report one byte. + * + * @param lead The lead byte. + * @return The sequence length in bytes, from one to four. + */ static int utf8Length(byte lead) { final int high = (lead & 0xFF) >>> 4; if (high < 0xC) { @@ -328,9 +338,15 @@ static int utf8Length(byte lead) { }; } - // Checks a single code point for well-formedness: correct trail-byte count and no unpaired - // surrogate or out-of-range value. The public tokenizer API encodes its own well-formed UTF-8, - // so this only guards direct byte-level use. + /** + * Checks a single code point for well-formedness: correct trail-byte count and no unpaired + * surrogate or out-of-range value. + * + * @param input The UTF-8 input buffer. + * @param from The offset of the lead byte. + * @param length The candidate sequence length. + * @return {@code true} when the sequence is malformed. + */ private static boolean isMalformed(byte[] input, int from, int length) { if ((input[from] & 0x80) == 0) { return false; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 5b39c395a9..959ecff05f 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -487,11 +487,12 @@ private void checkId(int id) { } } - // The embedded self-test samples, exposed for the parity tests. + /** {@return the embedded self-test input samples} */ List selfTestInputs() { return selfTestInputs; } + /** {@return the embedded self-test expected segmentations} */ List selfTestExpected() { return selfTestExpected; } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java index 49ffcc2909..c934150d07 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java @@ -24,13 +24,8 @@ * Viterbi segmentation under a unigram language model: of all ways to cover the normalized text * with vocabulary pieces, it finds the one with the highest total log-probability. * - *

This is a port of the reference implementation's optimized single-pass decoder, which - * exploits the unigram independence assumption to keep only the best path ending at each byte - * position instead of a full lattice. Characters no piece covers fall back to the unknown id - * with a fixed penalty below the lowest piece score, and user-defined symbols receive a - * length-based bonus score so they always win. Tie-breaking and score arithmetic follow the - * reference exactly, including its occasional re-basing of accumulated scores on very long - * inputs, because segmentation parity is asserted against it.

+ *

Characters no piece covers fall back to the unknown id with a fixed penalty below the lowest + * piece score, and user-defined symbols receive a length-based bonus score so they always win.

*/ final class UnigramEncoder { diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java index 6ebb080f5e..e830816e64 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java @@ -20,10 +20,9 @@ * A caller's text encoded as UTF-8, keeping the map from every byte offset back to the UTF-16 * offset it came from. * - *

The whole pipeline runs in byte space to match the reference implementation, but the spans - * reported to the caller must be UTF-16 offsets into the original {@code CharSequence}; this map - * converts them. An unpaired surrogate, which UTF-8 cannot represent, is encoded as U+FFFD, kept - * deterministic so that parity fixtures can cover it.

+ *

The pipeline runs in UTF-8 byte space, but the spans reported to the caller must be UTF-16 + * offsets into the original {@code CharSequence}; this map converts them. An unpaired surrogate, + * which UTF-8 cannot represent, is encoded as U+FFFD.

*/ final class Utf8Text { @@ -98,8 +97,6 @@ static Utf8Text of(CharSequence text) { c += charCount; } byteToChar[b] = charLength; - // The oversized buffers are kept and carried with an explicit length instead of being - // trimmed; the per-call copies were pure allocation traffic. return new Utf8Text(bytes, b, byteToChar, charLength); } From dea3031f2915cc31354aa9308f6e09072b483011 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 20:45:43 -0400 Subject: [PATCH 07/82] OPENNLP-1885: Tighten javadoc to contracts and document helpers and overrides Applies the review conventions from the OPENNLP-1869 review: class javadoc states the contracts instead of design narrative, every override carries inheritDoc with its null contract, and the private helpers are documented. --- .../opennlp/tools/tokenize/SubwordPiece.java | 11 ++-- .../tools/tokenize/SubwordTokenizer.java | 11 ++-- .../tools/tokenize/WordpieceEncoder.java | 5 ++ .../sentencepiece/DoubleArrayTrie.java | 4 +- .../sentencepiece/ModelProtoReader.java | 66 +++++++++++++++---- .../subword/sentencepiece/PieceTrie.java | 30 +++++++-- .../SentencePieceNormalizer.java | 25 +++++++ .../sentencepiece/SentencePieceTokenizer.java | 45 +++++++++++-- .../subword/sentencepiece/UnigramEncoder.java | 3 +- 9 files changed, 161 insertions(+), 39 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java index 7f790650cf..032d04f1c3 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java @@ -22,12 +22,11 @@ * One subword unit produced by a {@link SubwordTokenizer}, carrying both the vocabulary view * (the piece string and its id) and the exact place in the caller's text it came from. * - *

The piece string is in the tokenizer's internal, normalized form (for example, a leading - * word-boundary marker instead of a space), so it is generally not a substring of the input. - * {@code start} and {@code end} are UTF-16 offsets into the original input text, so the surface - * that produced this piece is {@code text.subSequence(start, end)}. Pieces that carry no surface - * of their own (control symbols, or the fill bytes of a byte-fallback expansion) report an empty - * span, {@code start == end}.

+ *

The piece string is in the tokenizer's normalized form, so it is generally not a substring of + * the input. {@code start} and {@code end} are UTF-16 offsets into the original text, so the + * surface that produced this piece is {@code text.subSequence(start, end)}. Pieces that carry no + * surface of their own, such as control symbols or the fill bytes of a byte-fallback expansion, + * report an empty span with {@code start == end}.

* * @param piece The piece in the vocabulary's normalized form; never null or empty. * @param id The vocabulary id of the piece. diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java index 5c371287f2..befcdd29ea 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java @@ -22,12 +22,11 @@ * Splits text into subword units against a fixed vocabulary, reporting for every unit its * vocabulary id and the exact span of the original text it covers. * - *

Subword tokenization is the input layer of modern sequence models: text is decomposed into - * pieces from a trained vocabulary so that any input, including words never seen in training, maps - * to a bounded id space. Unlike a linguistic {@link Tokenizer}, the segmentation is - * vocabulary-driven, and the pieces are in the model's normalized form rather than substrings of - * the input. The offsets carried by each {@link SubwordPiece} are what tie the two worlds - * together: they always refer to the caller's original text.

+ *

Subword tokenization splits text into pieces drawn from a trained vocabulary, so that any + * input, including words never seen in training, maps to a bounded id space. The segmentation is + * vocabulary-driven rather than linguistic, and each piece is in the model's normalized form, so a + * piece is generally not a substring of the input. The offsets carried by each + * {@link SubwordPiece} always refer to the caller's original text.

* *

Implementations are expected to be safe for concurrent use by multiple threads; any * implementation that is not must document it.

diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java index 65c73dab0c..ab016bbaa4 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java @@ -171,6 +171,11 @@ private static int requiredId(Map ids, String specialToken) { return id; } + /** + * {@inheritDoc} + * + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ @Override public List encode(CharSequence text) { if (text == null) { diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java index 967b525fe5..0ffb5e8be4 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java @@ -60,8 +60,8 @@ final class DoubleArrayTrie { * key matches. Values are non-negative, so the result is negative only on no-match. */ long longestPrefixMatch(byte[] key, int from, int to) { - // The JVM's own bounds checks guard the walk; a well-formed trie never leaves the array, - // so the translation below is the fail-loud path for corrupt data, not a hot branch. + // The JVM's own bounds checks guard the walk; the catch below translates an out-of-range unit + // reference from corrupt data into a loud failure. final int[] u = units; try { long result = -1; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java index 58f9dd83e1..4122746cb0 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java @@ -23,19 +23,11 @@ /** * Reads the binary {@code ModelProto} serialization of a SentencePiece {@code .model} file. * - *

The format is standard protocol-buffer wire encoding of one flat message - * ({@code sentencepiece_model.proto}, Apache License 2.0), so this reader walks the tag stream + *

The format is the standard protocol-buffer wire encoding of one flat message + * ({@code sentencepiece_model.proto}, Apache License 2.0). This reader walks the tag stream * directly and keeps only the fields inference needs: the pieces with scores and types, the - * normalizer spec, the handful of trainer-spec fields that change runtime behavior, and the - * embedded self-test samples. Unknown fields are skipped, malformed input fails loudly.

- * - *

This is a self-contained wire reader, not a performance optimization: model loading happens - * once and is not on any hot path. It exists so this module reads a protobuf-encoded file without - * adding a {@code protobuf-java} runtime dependency, which the project does not otherwise use, for - * a load-time parse of a schema that is stable in practice. If OpenNLP ever takes on a real - * protobuf dependency for other reasons, this hand-rolled reader should be retired in favor of it: - * generate from {@code sentencepiece_model.proto} (or parse via the descriptor API) and delete - * this class, since the dependency-avoidance rationale no longer holds.

+ * normalizer spec, the trainer-spec fields that change runtime behavior, and the embedded + * self-test samples. Unknown fields are skipped, and malformed input fails loudly.

*/ final class ModelProtoReader { @@ -82,6 +74,12 @@ static RawModel read(byte[] data) { return model; } + /** + * Parses one {@code SentencePiece} sub-message and appends its piece, score, and type. + * + * @param model The model to append to. + * @param end The exclusive end offset of the sub-message payload. + */ private void piece(RawModel model, int end) { String piece = null; float score = 0; @@ -107,6 +105,12 @@ private void piece(RawModel model, int end) { model.types.add(type); } + /** + * Parses the {@code TrainerSpec} sub-message, keeping the fields that change runtime behavior. + * + * @param model The model to populate. + * @param end The exclusive end offset of the sub-message payload. + */ private void trainerSpec(RawModel model, int end) { while (pos < end) { final long tag = varint(); @@ -120,6 +124,13 @@ private void trainerSpec(RawModel model, int end) { } } + /** + * Parses the {@code NormalizerSpec} sub-message: the precompiled character map and the + * whitespace-handling flags. + * + * @param model The model to populate. + * @param end The exclusive end offset of the sub-message payload. + */ private void normalizerSpec(RawModel model, int end) { while (pos < end) { final long tag = varint(); @@ -133,6 +144,13 @@ private void normalizerSpec(RawModel model, int end) { } } + /** + * Parses the {@code SelfTestData} sub-message, collecting the input and expected-segmentation + * sample pairs. + * + * @param model The model to populate. + * @param end The exclusive end offset of the sub-message payload. + */ private void selfTestData(RawModel model, int end) { while (pos < end) { final long tag = varint(); @@ -158,7 +176,15 @@ private void selfTestData(RawModel model, int end) { } } - // Returns the exclusive end offset of a length-delimited payload, verifying the wire type. + /** + * Reads the length prefix of a length-delimited field and returns the exclusive end offset of its + * payload. + * + * @param tag The field tag, whose wire type must be length-delimited. + * @return The exclusive end offset of the payload. + * @throws IllegalArgumentException Thrown if the wire type is wrong or the length runs past the + * input. + */ private int lenPayload(long tag) { if ((tag & 7) != WIRE_LEN) { throw malformed("field " + (tag >>> 3) + " is not length-delimited"); @@ -203,6 +229,13 @@ private byte[] bytes(int end) { return b; } + /** + * Reads a base-128 varint from the current position, advancing past it. + * + * @return The decoded value. + * @throws IllegalArgumentException Thrown if the input ends mid-varint or the varint exceeds 64 + * bits. + */ private long varint() { long value = 0; for (int shift = 0; shift < 64; shift += 7) { @@ -218,6 +251,13 @@ private long varint() { throw malformed("varint exceeds 64 bits"); } + /** + * Skips the value of an unrecognized field according to its wire type. + * + * @param tag The field tag. + * @throws IllegalArgumentException Thrown if the wire type is unsupported or the value runs past + * the input. + */ private void skip(long tag) { switch ((int) (tag & 7)) { case WIRE_VARINT -> varint(); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java index 37c2ef7912..e296e05b3b 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java @@ -22,12 +22,10 @@ /** * An immutable byte-level trie over vocabulary pieces, packed into flat arrays. * - *

Encoding walks it one byte at a time ({@link #step(int, byte)}) while scanning the input, so - * every piece that starts at a given input position is enumerated in one forward pass; this is the - * lattice-population step of subword segmentation. This step sits in the innermost loop of the - * encoder, so wide nodes (the root and the first level of a real vocabulary) dispatch through a - * 256-entry direct table, one load per byte, and narrow nodes scan their short sorted label slice - * linearly; both layouts enumerate identical transitions.

+ *

Encoding walks it one byte at a time ({@link #step(int, byte)}), so every piece that starts + * at a given input position is enumerated in one forward pass. Wide nodes dispatch through a + * 256-entry direct table and narrow nodes scan a short sorted label slice; both layouts enumerate + * identical transitions.

*/ final class PieceTrie { @@ -158,6 +156,15 @@ private static final class Builder { this.order = order; } + /** + * Counts the nodes and edges of the subtrie for the sorted key range {@code [from, to)} at the + * given depth. + * + * @param from The inclusive start index into {@code order}. + * @param to The exclusive end index into {@code order}. + * @param depth The byte depth this node partitions on. + * @throws IllegalArgumentException Thrown if a key is defined more than once. + */ void count(int from, int to, int depth) { nodeCount++; int i = from; @@ -182,6 +189,7 @@ void count(int from, int to, int depth) { } } + /** Allocates the packed arrays to the node and edge counts gathered by {@link #count}. */ void allocate() { childStart = new int[nodeCount + 1]; labels = new byte[edgeCount]; @@ -189,6 +197,16 @@ void allocate() { values = new int[nodeCount]; } + /** + * Fills the packed arrays for the sorted key range {@code [from, to)} at the given depth and + * returns the node id assigned to it. + * + * @param from The inclusive start index into {@code order}. + * @param to The exclusive end index into {@code order}. + * @param depth The byte depth this node partitions on. + * @return The id of the node created for this range. + * @throws IllegalArgumentException Thrown if a key is defined more than once. + */ int fill(int from, int to, int depth) { final int node = nextNode++; values[node] = -1; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java index 8c6ed5a4d7..29e8e5e895 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -127,6 +127,7 @@ private static final class Chunk { private int to; private int consumed; + /** {@return whether this chunk is exactly one ASCII space byte} */ boolean isSingleSpace() { return to - from == 1 && data[from] == ' '; } @@ -304,6 +305,15 @@ private void normalizePrefix(byte[] input, int inputLength, int from, Chunk chun chunk.consumed = charLength; } + /** + * Returns the byte length of the longest user-defined symbol that is a prefix of + * {@code input[from, inputLength)}. + * + * @param input The UTF-8 input buffer. + * @param inputLength The number of valid bytes in {@code input}. + * @param from The offset to match from. + * @return The matched length in bytes, or zero when no user-defined symbol matches. + */ private int longestUserDefinedMatch(byte[] input, int inputLength, int from) { int node = userDefinedMatcher.root(); int longest = 0; @@ -364,6 +374,14 @@ private static boolean isMalformed(byte[] input, int from, int length) { || length != minimalUtf8Length(codePoint); } + /** + * Decodes the code point of a UTF-8 sequence of the given length. + * + * @param input The UTF-8 input buffer. + * @param from The offset of the lead byte. + * @param length The sequence length in bytes, from one to four. + * @return The decoded code point. + */ private static int codePointAt(byte[] input, int from, int length) { return switch (length) { case 1 -> input[from] & 0x7F; @@ -375,6 +393,13 @@ private static int codePointAt(byte[] input, int from, int length) { }; } + /** + * Returns the number of bytes the shortest UTF-8 encoding of a code point uses, which detects + * overlong encodings. + * + * @param codePoint The code point. + * @return The minimal encoding length in bytes, from one to four. + */ private static int minimalUtf8Length(int codePoint) { if (codePoint < 0x80) { return 1; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 959ecff05f..dd39b894dc 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -41,10 +41,9 @@ * reference implementation piece for piece and id for id, which is what makes the produced ids * valid inputs for models trained against the same vocabulary.

* - *

Beyond parity, every piece carries the exact span of the caller's original text it came - * from, mapped back through the model's own normalizer. The normalizer is also exposed on its own - * through {@link OffsetAwareNormalizer}, so the model's text normalization can be reused as an - * offset-aware step outside of tokenization.

+ *

Every piece carries the exact span of the caller's original text it came from, mapped back + * through the model's own normalizer. That normalizer is also exposed through + * {@link OffsetAwareNormalizer} for reuse as an offset-aware step outside tokenization.

* *

Instances are immutable after loading and safe for concurrent use by multiple threads.

*/ @@ -247,6 +246,11 @@ public static SentencePieceTokenizer load(InputStream in) throws IOException { return new SentencePieceTokenizer(ModelProtoReader.read(in.readAllBytes())); } + /** + * {@inheritDoc} + * + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ @Override public List encode(CharSequence text) { if (text == null) { @@ -326,11 +330,21 @@ public List encode(CharSequence text) { return out; } + /** + * {@inheritDoc} + * + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ @Override public CharSequence normalize(CharSequence text) { return normalizeAligned(text).normalized(); } + /** + * {@inheritDoc} + * + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ @Override public AlignedText normalizeAligned(CharSequence text) { if (text == null) { @@ -376,6 +390,17 @@ public AlignedText normalizeAligned(CharSequence text) { return new AlignedText(text, normalized, builder.build(input.charLength())); } + /** + * Emits the pending alignment group as a replace run, preceded by a deletion for any original + * text skipped before it, and returns the advanced cursor. + * + * @param builder The alignment builder to append to. + * @param cursor The original-text offset reached so far. + * @param origStart The inclusive original-text start of the group. + * @param origEnd The exclusive original-text end of the group. + * @param chars The number of normalized chars in the group; zero flushes nothing. + * @return The original-text offset after the group. + */ private static int flushGroup(Alignment.Builder builder, int cursor, int origStart, int origEnd, int chars) { if (chars == 0) { @@ -480,6 +505,12 @@ public boolean isByte(int id) { return types[id] == TYPE_BYTE; } + /** + * Verifies that an id is a valid vocabulary id. + * + * @param id The id to check. + * @throws IllegalArgumentException Thrown if {@code id} is outside {@code [0, vocabularySize())}. + */ private void checkId(int id) { if (id < 0 || id >= pieces.length) { throw new IllegalArgumentException( @@ -507,6 +538,12 @@ List selfTestExpected() { } } + /** + * Parses a byte-fallback piece string of the form {@code <0xAB>} into its byte value. + * + * @param piece The piece string. + * @return The byte value in {@code [0, 255]}, or {@code -1} when the string is not a byte piece. + */ private static int parseBytePiece(String piece) { if (piece.length() != 6 || !piece.startsWith("<0x") || piece.charAt(5) != '>') { return -1; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java index c934150d07..c87662bef2 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java @@ -72,8 +72,7 @@ List encode(byte[] normalized, int size) { } // The best path ending at each byte position (exclusive end), interleaved as - // [startsAt, scoreBits, id] triples so a frontier update touches one cache line. Scores - // travel as raw float bits, a lossless round trip; all arithmetic happens on the floats. + // [startsAt, scoreBits, id] triples; scores travel as raw float bits, a lossless round trip. final int[] best = new int[3 * (size + 1)]; for (int i = 0; i <= size; i++) { best[3 * i] = -1; From 1f3883a677c7ee0b8f1b106f4bd132ca582d7ab4 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 13 Jul 2026 02:33:10 -0400 Subject: [PATCH 08/82] OPENNLP-1885: Declare serialVersionUID on SentencePieceTokenizer The tokenizer is Serializable through the OffsetAwareNormalizer contract but declared no serialVersionUID, which the compiler warns about. Added the serialver-computed value so it matches the convention used across the normalizer classes. --- .../opennlp/subword/sentencepiece/SentencePieceTokenizer.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index dd39b894dc..36d63c05e1 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -49,6 +49,10 @@ */ public final class SentencePieceTokenizer implements SubwordTokenizer, OffsetAwareNormalizer { + // Serializable through the OffsetAwareNormalizer contract; the model state is not itself + // serializable, so this only carries the serialver-computed value the convention expects. + private static final long serialVersionUID = -7114394869301531147L; + /** The segmentation algorithm a model was trained with. */ public enum Algorithm { /** Unigram language model, decoded by best-path search. */ From 976326a3f9cea29deb7d1244dfd362a283d24a31 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 13 Jul 2026 03:29:12 -0400 Subject: [PATCH 09/82] OPENNLP-1885: Trim residual commentary per review conventions --- .../tools/tokenize/SubwordTokenizer.java | 8 +++---- .../subword/sentencepiece/PieceTrie.java | 2 +- .../SentencePieceNormalizer.java | 8 +++---- .../sentencepiece/SentencePieceTokenizer.java | 22 +++++++------------ 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java index befcdd29ea..bd3cdf3984 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java @@ -22,11 +22,9 @@ * Splits text into subword units against a fixed vocabulary, reporting for every unit its * vocabulary id and the exact span of the original text it covers. * - *

Subword tokenization splits text into pieces drawn from a trained vocabulary, so that any - * input, including words never seen in training, maps to a bounded id space. The segmentation is - * vocabulary-driven rather than linguistic, and each piece is in the model's normalized form, so a - * piece is generally not a substring of the input. The offsets carried by each - * {@link SubwordPiece} always refer to the caller's original text.

+ *

The segmentation is vocabulary-driven rather than linguistic, and each piece is in the + * model's normalized form, so a piece is generally not a substring of the input. The offsets + * carried by each {@link SubwordPiece} always refer to the caller's original text.

* *

Implementations are expected to be safe for concurrent use by multiple threads; any * implementation that is not must document it.

diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java index e296e05b3b..795a251bc8 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java @@ -33,7 +33,7 @@ final class PieceTrie { static final int DEAD = -1; // A node dispatches through a 256-entry slice of directPool when it has more children than - // this; below it, a linear scan of the sorted label slice wins on memory and is branch-cheap. + // this; otherwise a linear scan of the sorted label slice is used. private static final int DIRECT_THRESHOLD = 8; // Per node: the slice [childStart[n], childStart[n + 1]) of labels/childNodes, and the piece id diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java index 29e8e5e895..4d5589514e 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -43,8 +43,7 @@ final class SentencePieceNormalizer { private final boolean treatWhitespaceAsSuffix; private final PieceTrie userDefinedMatcher; // For each possible first byte, whether any character-map rule or user-defined symbol starts - // with it. A clear bit proves normalizePrefix would pass the byte through raw, which lets the - // scan skip the whole prefix machinery for plain ASCII text. + // with it; a clear bit means normalizePrefix passes the byte through raw. private final boolean[] ruleLead = new boolean[256]; /** @@ -118,7 +117,6 @@ record Normalized(byte[] bytes, int length, int[] normToOrig) { /** * One normalization step: {@code consumed} input bytes produced {@code data[from, to)}. The data * array is the input itself (pass-through), the replacement blob, or the replacement character. - * A single mutable scratch is refilled per chunk so the scan allocates nothing per code point. */ private static final class Chunk { @@ -176,8 +174,8 @@ Normalized normalize(byte[] input, int inputLength) { boolean isPrevSpace = removeExtraWhitespaces; while (from < inputLength) { final int lead = input[from] & 0xFF; - // Fast path: an ASCII byte no rule starts with passes through raw; the chunk would be - // the byte itself, no leading-space stripping applies, and it does not end in a space. + // An ASCII byte no rule starts with passes through raw: the chunk is the byte itself, no + // leading-space stripping applies, and it does not end in a space. if (lead < 0x80 && lead != ' ' && !ruleLead[lead]) { normalized.append(input[from]); normToOrig.append(consumed); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 36d63c05e1..961bb58b3c 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -33,24 +33,20 @@ import opennlp.tools.util.normalizer.OffsetAwareNormalizer; /** - * A {@link SubwordTokenizer} over a trained SentencePiece model file, implemented purely in Java. - * - *

A {@code .model} file is self-contained: it carries the vocabulary with piece scores and - * types, the segmentation algorithm (unigram language model or byte-pair encoding), and the text - * normalizer the model was trained with. This class runs all three, so its output matches the - * reference implementation piece for piece and id for id, which is what makes the produced ids - * valid inputs for models trained against the same vocabulary.

+ * A {@link SubwordTokenizer} over a trained SentencePiece {@code .model} file, implemented purely + * in Java. The file carries the vocabulary with piece scores and types, the segmentation algorithm + * (unigram language model or byte-pair encoding), and the text normalizer, all of which this class + * runs. * *

Every piece carries the exact span of the caller's original text it came from, mapped back - * through the model's own normalizer. That normalizer is also exposed through - * {@link OffsetAwareNormalizer} for reuse as an offset-aware step outside tokenization.

+ * through the model's own normalizer, which is also exposed through {@link OffsetAwareNormalizer} + * for reuse outside tokenization.

* *

Instances are immutable after loading and safe for concurrent use by multiple threads.

*/ public final class SentencePieceTokenizer implements SubwordTokenizer, OffsetAwareNormalizer { - // Serializable through the OffsetAwareNormalizer contract; the model state is not itself - // serializable, so this only carries the serialver-computed value the convention expects. + // Serializable through the OffsetAwareNormalizer contract. private static final long serialVersionUID = -7114394869301531147L; /** The segmentation algorithm a model was trained with. */ @@ -280,9 +276,7 @@ public List encode(CharSequence text) { for (final Segment segment : segments) { final boolean isUnk = segment.id() == unkId; final boolean isControl = types[segment.id()] == TYPE_CONTROL; - // A non-unknown segment's bytes are exactly the vocabulary piece's bytes (that is what - // the match meant), so the vocabulary string is reused; only unknown segments carry - // surface content that needs decoding. + // A non-unknown segment reuses its vocabulary string; only unknown segments need decoding. final String piece = isUnk ? new String(norm, segment.from(), segment.to() - segment.from(), StandardCharsets.UTF_8) : pieces[segment.id()]; From bf3cacff4aac1b6f098286ea51b21a913b50c0e8 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 14 Jul 2026 10:42:20 -0400 Subject: [PATCH 10/82] OPENNLP-1885: Document subword tokenization in the manual Adds a Subword Tokenization section to the Tokenizer chapter: the SubwordTokenizer contract and its original-text span guarantee, loading and using a SentencePiece model including the OffsetAwareNormalizer face, and the WordpieceEncoder pipeline with its vocab.txt construction and special-token framing. --- opennlp-docs/src/docbkx/tokenizer.xml | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index cd1d8a2ddf..b19ec7ad9f 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -539,4 +539,79 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { +
+ Subword Tokenization + + Neural models usually operate on subword units drawn from a fixed vocabulary rather than + on words. The SubwordTokenizer interface in + opennlp.tools.tokenize covers this case: encode splits text into + pieces, and every returned SubwordPiece carries the piece string in the + model's normalized form, its vocabulary id, and the exact span of the original text it + came from. Because the segmentation runs after the model's own normalization, a piece is + generally not a substring of the input; the spans always refer to the caller's original + text, so annotations computed over the pieces can be mapped back without guesswork. The + encodeToIds and encodeToPieces methods return just the ids or + the piece strings when the spans are not needed. Implementations are safe for concurrent + use by multiple threads. + +
+ SentencePiece + + SentencePieceTokenizer in the opennlp-subword artifact + (package opennlp.subword.sentencepiece) runs a trained SentencePiece + .model file purely in Java, with no native library. The file itself + carries everything the class needs: the vocabulary with piece scores and types, the + segmentation algorithm (unigram language model or byte-pair encoding), and the text + normalizer the model was trained with. Loading is a one-time cost and the resulting + instance is immutable, so one tokenizer can be shared by any number of threads. + + id " + piece.id() + + ", original text [" + piece.start() + ", " + piece.end() + ")"); +} + +int[] ids = tokenizer.encodeToIds("Ready for the embedding layer.");]]> + + The vocabulary can be inspected through vocabularySize, + idToPiece, pieceToId, and score, and the + algorithm method reports whether the model uses the unigram or the + byte-pair encoding algorithm. + + + The model's own normalizer is also exposed directly: + SentencePieceTokenizer implements OffsetAwareNormalizer, so + normalize applies the model's normalization rules to arbitrary text and + normalizeAligned additionally returns the character alignment described + in . This is useful when other processing must see + text exactly as the subword model does. + +
+
+ WordPiece + + WordpieceEncoder in opennlp.tools.tokenize runs the full + BERT tokenization pipeline: control character removal, whitespace normalization, CJK + isolation, optional lower casing with accent stripping, punctuation isolation, and + greedy longest-match wordpiece segmentation. It is constructed from the model's + vocabulary, conventionally the lines of a vocab.txt file, where a piece's + id is its line number. + + vocabulary = Files.readAllLines(Path.of("vocab.txt")); + +WordpieceEncoder encoder = new WordpieceEncoder(vocabulary); // uncased model +WordpieceEncoder cased = new WordpieceEncoder(vocabulary, false); // cased model + +List pieces = encoder.encode("OpenNLP encodes text for BERT models."); +int[] ids = encoder.encodeToIds("OpenNLP encodes text for BERT models.");]]> + + Every encoding is framed by the classification and separator pieces, which carry empty + spans at the text's boundaries, and words the vocabulary cannot cover become the + unknown piece. Models with other special tokens or with non-contiguous ids are + supported through the constructors taking explicit special tokens or a piece-to-id + map. + +
+
From 81b15814b08bb4148e11ac82c3199057d8e40f67 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 06:27:30 -0400 Subject: [PATCH 11/82] OPENNLP-1885: Make the tokenizer graph serializable with computed UIDs, name the format constants, document every helper --- .../tools/tokenize/WordpieceEncoder.java | 77 ++++++++++- .../java/opennlp/dl/CreateTokenizerTest.java | 3 +- .../tools/tokenize/ReferenceBertPipeline.java | 2 + ...ordpieceEncoderReferenceSequencesTest.java | 130 +++++++----------- .../tools/tokenize/WordpieceEncoderTest.java | 47 ++++--- opennlp-docs/src/docbkx/tokenizer.xml | 4 +- .../subword/sentencepiece/BpeEncoder.java | 40 +++--- .../subword/sentencepiece/ByteBuilder.java | 8 +- .../sentencepiece/DoubleArrayTrie.java | 32 ++++- .../subword/sentencepiece/IntBuilder.java | 8 +- .../sentencepiece/ModelProtoReader.java | 115 +++++++++++++--- .../subword/sentencepiece/PieceTrie.java | 68 +++++++-- .../SentencePieceNormalizer.java | 41 ++---- .../sentencepiece/SentencePieceTokenizer.java | 44 ++++-- .../subword/sentencepiece/UnigramEncoder.java | 15 +- .../subword/sentencepiece/Utf8Text.java | 8 ++ .../sentencepiece/SentencePieceFixtures.java | 125 +++++++++++++++++ .../SentencePieceModelValidationTest.java | 6 +- .../SentencePieceParityTest.java | 65 +-------- .../SentencePieceRealModelEvalTest.java | 50 +------ ...ntencePieceTokenizerSerializationTest.java | 71 ++++++++++ 21 files changed, 653 insertions(+), 306 deletions(-) create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java index ab016bbaa4..b130f45dca 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java @@ -40,7 +40,8 @@ *

Ids follow the line-number convention of BERT {@code vocab.txt} files: with the list * constructors a piece's id is its index, and with the map constructor the ids are given * explicitly. The classification, separator, and unknown tokens must all be present in the - * vocabulary, because every emitted piece must have an id.

+ * vocabulary, because every emitted piece must have an id. Vocabulary entries starting with + * {@code ##} are continuation pieces, matching a word's interior rather than its start.

* *

Instances are immutable and safe for concurrent use by multiple threads.

* @@ -48,6 +49,10 @@ */ public final class WordpieceEncoder implements SubwordTokenizer { + // The wordpiece vocabulary convention: a piece with this prefix continues the current word, + // so it can only match after the word's first piece. + private static final String CONTINUATION_PREFIX = "##"; + // The reference implementation's limit: longer words become the unknown piece. private static final int MAX_WORD_CHARACTERS = 100; @@ -144,6 +149,15 @@ public WordpieceEncoder(Map vocabularyIds, boolean lowerCase, this.unknownId = requiredId(byPiece, unknownToken); } + /** + * Converts an ordered vocabulary list into the piece-to-id mapping, assigning each piece its + * index as the id. + * + * @param vocabulary The ordered vocabulary. + * @return The piece-to-id mapping. + * @throws IllegalArgumentException Thrown if the list is null or contains a null or duplicate + * entry. + */ private static Map byPiece(List vocabulary) { if (vocabulary == null) { throw new IllegalArgumentException("The vocabulary must not be null."); @@ -162,6 +176,14 @@ private static Map byPiece(List vocabulary) { return byPiece; } + /** + * Looks up the id of a special token that must be present in the vocabulary. + * + * @param ids The piece-to-id mapping. + * @param specialToken The token to look up. + * @return The token's id. + * @throws IllegalArgumentException Thrown if the token is not in the vocabulary. + */ private static int requiredId(Map ids, String specialToken) { final Integer id = ids.get(specialToken); if (id == null) { @@ -171,11 +193,7 @@ private static int requiredId(Map ids, String specialToken) { return id; } - /** - * {@inheritDoc} - * - * @throws IllegalArgumentException Thrown if {@code text} is null. - */ + /** {@inheritDoc} */ @Override public List encode(CharSequence text) { if (text == null) { @@ -236,7 +254,7 @@ private void encodeWord(MappedText mapped, int from, int to, List while (start < end) { String substring = new String(mapped.chars, start, end - start); if (start > from) { - substring = "##" + substring; + substring = CONTINUATION_PREFIX + substring; } if (vocabulary.contains(substring)) { wordPieces.add(new SubwordPiece(substring, ids.get(substring), @@ -268,12 +286,24 @@ private static final class MappedText { private int[] ends; private int length; + /** + * Instantiates an empty mapped text. + * + * @param capacity The initial capacity hint in chars. + */ private MappedText(int capacity) { chars = new char[capacity]; starts = new int[capacity]; ends = new int[capacity]; } + /** + * Appends one char with the original-text range it came from. + * + * @param c The char to append. + * @param originalStart The inclusive original-text start of the char. + * @param originalEnd The exclusive original-text end of the char. + */ private void add(char c, int originalStart, int originalEnd) { if (length == chars.length) { final int capacity = Math.max(16, length * 2); @@ -287,6 +317,13 @@ private void add(char c, int originalStart, int originalEnd) { length++; } + /** + * Appends every char of a string, all sharing one original-text range. + * + * @param s The string to append. + * @param originalStart The inclusive original-text start shared by all chars. + * @param originalEnd The exclusive original-text end shared by all chars. + */ private void add(String s, int originalStart, int originalEnd) { for (int i = 0; i < s.length(); i++) { add(s.charAt(i), originalStart, originalEnd); @@ -385,8 +422,20 @@ private static MappedText lowerCaseAndStripAccents(MappedText in) { return out; } + /** + * Lower cases and accent-strips one non-space run, emitting per-character ranges when the + * transformation is reproducible per code point and the run's full range otherwise. + * + * @param in The input text with per-character ranges. + * @param from The inclusive start of the run in {@code in}. + * @param to The exclusive end of the run in {@code in}. + * @param out The output text to append to. + */ private static void transformRun(MappedText in, int from, int to, MappedText out) { final String run = new String(in.chars, from, to - from); + // Locale.ROOT lower casing is the reference behavior of BERT's do_lower_case: the reference + // pipeline applies the full locale-independent Unicode case mappings (including one-to-many + // ones like the dotted capital I), which a per-code-point mapping cannot reproduce. final String content = stripAccents(run.toLowerCase(Locale.ROOT)); // Rerun per code point to learn how many output chars each input code point produces. @@ -419,6 +468,13 @@ private static void transformRun(MappedText in, int from, int to, MappedText out } } + /** + * Removes combining marks after NFD decomposition, the accent stripping of BERT's + * {@code do_lower_case} mode. + * + * @param text The text to strip. + * @return The text without non-spacing marks. + */ private static String stripAccents(String text) { final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); final StringBuilder stripped = new StringBuilder(decomposed.length()); @@ -430,6 +486,13 @@ private static String stripAccents(String text) { return stripped.toString(); } + /** + * Reads the code point at an index, joining a surrogate pair when one starts there. + * + * @param text The text to read from. + * @param index The char index to read at. + * @return The code point at {@code index}. + */ private static int codePointAt(MappedText text, int index) { final char c = text.chars[index]; if (Character.isHighSurrogate(c) && index + 1 < text.length diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java index 5131ae84e7..5ae7e3ebee 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/CreateTokenizerTest.java @@ -101,7 +101,8 @@ void testRejectsRobertaVocabularyWithoutUnknownToken() { final Map vocab = robertaVocab(); vocab.remove(WordpieceTokenizer.ROBERTA_UNK_TOKEN); - assertThrows(IllegalArgumentException.class, () -> AbstractDL.createPipelineTokenizer(vocab, false)); + assertThrows(IllegalArgumentException.class, + () -> AbstractDL.createPipelineTokenizer(vocab, false)); assertThrows(IllegalArgumentException.class, () -> AbstractDL.createWordpieceTokenizer(vocab)); } diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java index 867c28dbfa..a07d6faafc 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java @@ -47,6 +47,8 @@ private String normalize(String text) { String normalized = cleanText(text); normalized = isolateCjkCharacters(normalized); if (lowerCase) { + // Locale.ROOT lower casing is the reference behavior of BERT's do_lower_case: the full + // locale-independent Unicode case mappings, including one-to-many ones. normalized = stripAccents(normalized.toLowerCase(Locale.ROOT)); } return BertNormalization.isolatePunctuation(normalized); diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java index 68e8f219f7..064d644a57 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java @@ -17,13 +17,17 @@ package opennlp.tools.tokenize; import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; /** - * The reference token sequences of the removed full-pipeline {@code Tokenizer}, re-asserted - * against {@link WordpieceEncoder}. + * Reference token-sequence expectations for {@link WordpieceEncoder}, covering lower casing, + * accent stripping, punctuation and CJK isolation, and text cleaning. *

* All expected token sequences in this test were generated with the HuggingFace * {@code tokenizers} reference implementation ({@code BertWordPieceTokenizer}) @@ -43,92 +47,56 @@ public class WordpieceEncoderReferenceSequencesTest { "\u6211", "\u7231", // CJK "natural", "language", "processing"); - @Test - void testLowerCasesCapitalizedWords() { - final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - final String[] tokens = - encoder.encodeToPieces("The quick brown fox jumps over the lazy dog."); - - final String[] expected = {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over", - "the", "lazy", "dog", ".", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testLowerCasesBeforeWordpieceSplitting() { - final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - final String[] tokens = encoder.encodeToPieces("Embeddings"); - - final String[] expected = {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testStripsAccentsButKeepsNonCombiningCharacters() { - final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped; - // the sharp s is not a combining mark and must survive, leaving an OOV token. - final String[] tokens = encoder.encodeToPieces("W\u00fcrttemberg Stra\u00dfe"); - - final String[] expected = {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); + /** + * The reference input and expected-sequence pairs, one argument set per pipeline behavior. + * + * @return The (input, expected pieces) pairs. + */ + static Stream referenceSequences() { + return Stream.of( + // Lower cases capitalized words. + Arguments.of("The quick brown fox jumps over the lazy dog.", + new String[] {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over", + "the", "lazy", "dog", ".", "[SEP]"}), + // Lower cases before wordpiece splitting. + Arguments.of("Embeddings", + new String[] {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"}), + // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped; + // the sharp s is not a combining mark and must survive, leaving an OOV token. + Arguments.of("W\u00fcrttemberg Stra\u00dfe", + new String[] {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"}), + // Splits punctuation runs into single characters. + Arguments.of("Wait... what?!", + new String[] {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"}), + // Splits apostrophes as punctuation. + Arguments.of("don't", + new String[] {"[CLS]", "don", "'", "t", "[SEP]"}), + // Isolates CJK ideographs into single-character pieces. + Arguments.of("\u6211\u7231natural language processing", + new String[] {"[CLS]", "\u6211", "\u7231", "natural", "language", + "processing", "[SEP]"}), + // Tab and no-break space are whitespace; the NUL character is removed, + // joining "brown" and "fox" into one out-of-vocabulary token. + Arguments.of("the\tquick\u00a0brown\u0000fox", + new String[] {"[CLS]", "the", "quick", "[UNK]", "[SEP]"}), + // The reference implementation treats all C* categories as control + // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn) + // are removed, joining the surrounding text into one OOV token. + Arguments.of("fox\ue000jumps and fox\ufdd0jumps", + new String[] {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"})); } - @Test - void testSplitsPunctuationRunsIntoSingleCharacters() { + @ParameterizedTest + @MethodSource("referenceSequences") + void testEncodesTheReferenceSequence(String input, String[] expected) { final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - final String[] tokens = encoder.encodeToPieces("Wait... what?!"); - - final String[] expected = {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testSplitsApostrophesAsPunctuation() { - final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - final String[] tokens = encoder.encodeToPieces("don't"); - - final String[] expected = {"[CLS]", "don", "'", "t", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testIsolatesCjkIdeographs() { - final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - final String[] tokens = encoder.encodeToPieces("\u6211\u7231natural language processing"); - - final String[] expected = {"[CLS]", "\u6211", "\u7231", "natural", "language", - "processing", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testCleansControlCharactersAndNormalizesWhitespace() { - final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - // Tab and no-break space are whitespace; the NUL character is removed, - // joining "brown" and "fox" into one out-of-vocabulary token. - final String[] tokens = encoder.encodeToPieces("the\tquick\u00a0brown\u0000fox"); - - final String[] expected = {"[CLS]", "the", "quick", "[UNK]", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testRemovesPrivateUseAndUnassignedCharacters() { - final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); - // The reference implementation treats all C* categories as control - // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn) - // are removed, joining the surrounding text into one OOV token. - final String[] tokens = encoder.encodeToPieces("fox\ue000jumps and fox\ufdd0jumps"); - - final String[] expected = {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); + Assertions.assertArrayEquals(expected, encoder.encodeToPieces(input), + "sequence broke on: " + input); } @Test void testRejectsNullSpecialTokens() { - // The encoder's contract throws IllegalArgumentException where the removed class threw - // NullPointerException. + // The encoder's contract throws IllegalArgumentException for null special tokens. Assertions.assertThrows(IllegalArgumentException.class, () -> new WordpieceEncoder(VOCABULARY, true, null, "[SEP]", "[UNK]")); Assertions.assertThrows(IllegalArgumentException.class, diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java index 610bba5ed5..cf545b183f 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java @@ -20,8 +20,12 @@ import java.util.HashSet; import java.util.List; import java.util.Random; +import java.util.stream.Stream; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -55,11 +59,13 @@ private static void assertPiece(SubwordPiece piece, String expectedPiece, int ex assertEquals(expectedEnd, piece.end(), "end of " + piece); } - @Test - void testPieceSequenceMatchesTheReferencePipelineOnCuratedInputs() { - final ReferenceBertPipeline reference = new ReferenceBertPipeline(new HashSet<>(VOCAB), true); - final WordpieceEncoder encoder = uncased(); - final String[] inputs = { + /** + * The curated parity inputs, each exercising a normalization step of the pipeline. + * + * @return The inputs. + */ + static Stream curatedInputs() { + return Stream.of( "", " ", "Hello, WORLD!", @@ -81,12 +87,16 @@ void testPieceSequenceMatchesTheReferencePipelineOnCuratedInputs() { "\uD83D\uDE00", "!!!", "a".repeat(101), - "he said: \u00ABhello\u00BB.", - }; - for (final String input : inputs) { - assertArrayEquals(reference.tokenize(input), encoder.encodeToPieces(input), - "parity broke on: " + input); - } + "he said: \u00ABhello\u00BB."); + } + + @ParameterizedTest + @MethodSource("curatedInputs") + void testPieceSequenceMatchesTheReferencePipelineOnCuratedInputs(String input) { + final ReferenceBertPipeline reference = new ReferenceBertPipeline(new HashSet<>(VOCAB), true); + final WordpieceEncoder encoder = uncased(); + assertArrayEquals(reference.tokenize(input), encoder.encodeToPieces(input), + "parity broke on: " + input); } @Test @@ -193,14 +203,13 @@ void testCasedEncoderKeepsCase() { assertPiece(pieces.get(2), "hello", 4, 6, 11); } - @Test - void testEmptyAndBlankTextEncodeToTheFramePiecesOnly() { - for (final String input : new String[] {"", " "}) { - final List pieces = uncased().encode(input); - assertEquals(2, pieces.size()); - assertPiece(pieces.get(0), "[CLS]", 2, 0, 0); - assertPiece(pieces.get(1), "[SEP]", 3, input.length(), input.length()); - } + @ParameterizedTest + @ValueSource(strings = {"", " "}) + void testEmptyAndBlankTextEncodeToTheFramePiecesOnly(String input) { + final List pieces = uncased().encode(input); + assertEquals(2, pieces.size(), "frame pieces broke on <" + input + ">"); + assertPiece(pieces.get(0), "[CLS]", 2, 0, 0); + assertPiece(pieces.get(1), "[SEP]", 3, input.length(), input.length()); } @Test diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index b19ec7ad9f..b6b27c5c16 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -551,8 +551,8 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { generally not a substring of the input; the spans always refer to the caller's original text, so annotations computed over the pieces can be mapped back without guesswork. The encodeToIds and encodeToPieces methods return just the ids or - the piece strings when the spans are not needed. Implementations are safe for concurrent - use by multiple threads. + the piece strings when the spans are not needed. Implementations are expected to be safe + for concurrent use by multiple threads.

SentencePiece diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java index c6ad75542c..0a08c980eb 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java @@ -16,6 +16,7 @@ */ package opennlp.subword.sentencepiece; +import java.io.Serializable; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; @@ -31,7 +32,9 @@ *

Only pieces of the normal, user-defined, and unused types participate in merges; a merge that * lands on an unused piece is re-segmented back into its constituents.

*/ -final class BpeEncoder { +final class BpeEncoder implements Serializable { + + private static final long serialVersionUID = -57799941356582785L; private static final int MAX_RESEGMENT_DEPTH = 100; @@ -77,8 +80,12 @@ private record Pair(int left, int right, float score, int size) { * @param normalized The buffer holding the normalized UTF-8 bytes; must not be null. * @param size The number of valid bytes in {@code normalized}. * @return The segments covering all bytes, in text order. + * @throws IllegalArgumentException Thrown if {@code normalized} is null. */ List encode(byte[] normalized, int size) { + if (normalized == null) { + throw new IllegalArgumentException("The normalized buffer must not be null."); + } if (size == 0) { return List.of(); } @@ -92,7 +99,7 @@ List encode(byte[] normalized, int size) { while (position < size) { int matched = 0; if (userDefinedMatcher != null) { - matched = longestUserDefinedMatch(normalized, size, position); + matched = userDefinedMatcher.longestMatch(normalized, size, position); } final boolean frozen = matched > 0; final int length = frozen ? matched @@ -158,6 +165,20 @@ List encode(byte[] normalized, int size) { return output; } + /** + * Offers the adjacent symbol pair {@code (left, right)} as a merge candidate: the pair joins + * the agenda only when the concatenation is a mergeable vocabulary piece, and a merge landing + * on an unused piece is remembered in {@code revMerge} for later re-segmentation. + * + * @param normalized The buffer holding the normalized UTF-8 bytes. + * @param from Per symbol, the inclusive start offset in {@code normalized}. + * @param to Per symbol, the exclusive end offset in {@code normalized}. + * @param freeze Per symbol, whether it is a user-defined symbol excluded from merging. + * @param left The index of the left symbol, or {@code -1} for none. + * @param right The index of the right symbol, or {@code -1} for none. + * @param agenda The merge agenda to add to. + * @param revMerge The map from a merged piece to its two constituents. + */ private void maybeAddPair(byte[] normalized, int[] from, int[] to, boolean[] freeze, int left, int right, PriorityQueue agenda, Map revMerge) { @@ -207,19 +228,4 @@ private int resegment(String piece, int consumed, int depth, Map= 0) { - longest = i - from + 1; - } - } - return longest; - } } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java index 95a02fddd5..ef08bb624a 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java @@ -68,9 +68,15 @@ int length() { /** * Shrinks the valid length. * - * @param newLength The new length, not greater than the current length. + * @param newLength The new length, not negative and not greater than the current length. + * @throws IllegalArgumentException Thrown if {@code newLength} is negative or greater than the + * current length. */ void truncate(int newLength) { + if (newLength < 0 || newLength > length) { + throw new IllegalArgumentException( + "The new length " + newLength + " is outside [0, " + length + "]."); + } length = newLength; } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java index 0ffb5e8be4..28eefd3f1f 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java @@ -16,6 +16,8 @@ */ package opennlp.subword.sentencepiece; +import java.io.Serializable; + /** * Read-only lookup over a serialized Darts-clone double-array trie, the dictionary format * embedded in a SentencePiece model's precompiled character map. @@ -25,8 +27,23 @@ * prefix match is needed here, so this walks the byte key once and remembers the last accepting * state. Out-of-range unit references, which a well-formed trie never produces, fail loudly * rather than reading arbitrary memory.

+ * + * @see Darts-clone */ -final class DoubleArrayTrie { +final class DoubleArrayTrie implements Serializable { + + private static final long serialVersionUID = -1572336116472261588L; + + // A non-leaf unit stores its transition label in the low 8 bits and the leaf flag in the sign + // bit. Key bytes are in [0, 255] with the sign bit clear, so comparing (unit & this mask) + // against a key byte both matches the label and rejects leaf units in one test. + private static final int LEAF_FLAG_AND_LABEL_MASK = 0x800000FF; + + // A leaf unit stores the key's value in its low 31 bits; the sign bit is the leaf flag. + private static final int LEAF_VALUE_MASK = 0x7FFFFFFF; + + // Bit 8 of a non-leaf unit marks that one of its children is a leaf holding this key's value. + private static final int HAS_LEAF_BIT = 8; private final int[] units; @@ -72,12 +89,12 @@ long longestPrefixMatch(byte[] key, int from, int to) { final int b = key[i] & 0xFF; nodePos ^= b; unit = u[nodePos]; - if ((unit & 0x800000FF) != b) { + if ((unit & LEAF_FLAG_AND_LABEL_MASK) != b) { return result; } nodePos ^= offset(unit); - if (((unit >>> 8) & 1) == 1) { - final int value = u[nodePos] & 0x7FFFFFFF; + if (((unit >>> HAS_LEAF_BIT) & 1) == 1) { + final int value = u[nodePos] & LEAF_VALUE_MASK; result = ((long) value << 32) | (i - from + 1); } } @@ -101,11 +118,14 @@ boolean hasTransitionFromRoot(int b) { if (nodePos < 0 || nodePos >= units.length) { return false; } - return (units[nodePos] & 0x800000FF) == b; + return (units[nodePos] & LEAF_FLAG_AND_LABEL_MASK) == b; } /** - * Returns the offset from a unit to its children, as encoded by Darts-clone. + * Returns the offset from a unit to its children, as encoded by Darts-clone: bits 10 to 30 hold + * the raw offset, and bit 9 is an extension flag that scales it by 256 for far-away children. + * The expression {@code (unit & (1 << 9)) >>> 6} evaluates to 8 exactly when bit 9 is set, so + * the raw offset is shifted left by either 0 or 8 bits. * * @param unit The unit word. * @return The child offset. diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java index 25dad2ec92..2da34c29b6 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java @@ -67,9 +67,15 @@ int length() { /** * Shrinks the valid length. * - * @param newLength The new length, not greater than the current length. + * @param newLength The new length, not negative and not greater than the current length. + * @throws IllegalArgumentException Thrown if {@code newLength} is negative or greater than the + * current length. */ void truncate(int newLength) { + if (newLength < 0 || newLength > length) { + throw new IllegalArgumentException( + "The new length " + newLength + " is outside [0, " + length + "]."); + } length = newLength; } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java index 4122746cb0..c4c2a1472e 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java @@ -28,6 +28,10 @@ * directly and keeps only the fields inference needs: the pieces with scores and types, the * normalizer spec, the trainer-spec fields that change runtime behavior, and the embedded * self-test samples. Unknown fields are skipped, and malformed input fails loudly.

+ * + * @see + * sentencepiece_model.proto */ final class ModelProtoReader { @@ -37,9 +41,43 @@ final class ModelProtoReader { private static final int WIRE_LEN = 2; private static final int WIRE_FIXED32 = 5; + // Field numbers of the ModelProto message in sentencepiece_model.proto. + private static final int FIELD_MODEL_PIECES = 1; + private static final int FIELD_MODEL_TRAINER_SPEC = 2; + private static final int FIELD_MODEL_NORMALIZER_SPEC = 3; + private static final int FIELD_MODEL_SELF_TEST_DATA = 4; + + // Field numbers of the ModelProto.SentencePiece sub-message. + private static final int FIELD_PIECE_PIECE = 1; + private static final int FIELD_PIECE_SCORE = 2; + private static final int FIELD_PIECE_TYPE = 3; + + // Field numbers of the TrainerSpec sub-message. + private static final int FIELD_TRAINER_MODEL_TYPE = 3; + private static final int FIELD_TRAINER_TREAT_WHITESPACE_AS_SUFFIX = 24; + private static final int FIELD_TRAINER_BYTE_FALLBACK = 35; + private static final int FIELD_TRAINER_UNK_ID = 40; + + // Field numbers of the NormalizerSpec sub-message. + private static final int FIELD_NORMALIZER_PRECOMPILED_CHARSMAP = 2; + private static final int FIELD_NORMALIZER_ADD_DUMMY_PREFIX = 3; + private static final int FIELD_NORMALIZER_REMOVE_EXTRA_WHITESPACES = 4; + private static final int FIELD_NORMALIZER_ESCAPE_WHITESPACES = 5; + + // Field numbers of the SelfTestData sub-message and its Sample entries. + private static final int FIELD_SELF_TEST_SAMPLES = 1; + private static final int FIELD_SAMPLE_INPUT = 1; + private static final int FIELD_SAMPLE_EXPECTED = 2; + private final byte[] data; private int pos; + /** + * Prepares a reader positioned at the start of the given bytes; {@link #read(byte[])} drives + * the actual parse. + * + * @param data The raw bytes of a {@code .model} file. + */ private ModelProtoReader(byte[] data) { this.data = data; } @@ -61,10 +99,10 @@ static RawModel read(byte[] data) { final long tag = reader.varint(); final int field = (int) (tag >>> 3); switch (field) { - case 1 -> reader.piece(model, reader.lenPayload(tag)); - case 2 -> reader.trainerSpec(model, reader.lenPayload(tag)); - case 3 -> reader.normalizerSpec(model, reader.lenPayload(tag)); - case 4 -> reader.selfTestData(model, reader.lenPayload(tag)); + case FIELD_MODEL_PIECES -> reader.piece(model, reader.lenPayload(tag)); + case FIELD_MODEL_TRAINER_SPEC -> reader.trainerSpec(model, reader.lenPayload(tag)); + case FIELD_MODEL_NORMALIZER_SPEC -> reader.normalizerSpec(model, reader.lenPayload(tag)); + case FIELD_MODEL_SELF_TEST_DATA -> reader.selfTestData(model, reader.lenPayload(tag)); default -> reader.skip(tag); } } @@ -87,9 +125,9 @@ private void piece(RawModel model, int end) { while (pos < end) { final long tag = varint(); switch ((int) (tag >>> 3)) { - case 1 -> piece = utf8(lenPayload(tag)); - case 2 -> score = fixed32Float(tag); - case 3 -> type = (int) varintOf(tag); + case FIELD_PIECE_PIECE -> piece = utf8(lenPayload(tag)); + case FIELD_PIECE_SCORE -> score = fixed32Float(tag); + case FIELD_PIECE_TYPE -> type = (int) varintOf(tag); default -> skip(tag); } } @@ -115,10 +153,11 @@ private void trainerSpec(RawModel model, int end) { while (pos < end) { final long tag = varint(); switch ((int) (tag >>> 3)) { - case 3 -> model.modelType = (int) varintOf(tag); - case 24 -> model.treatWhitespaceAsSuffix = varintOf(tag) != 0; - case 35 -> model.byteFallback = varintOf(tag) != 0; - case 40 -> model.unkId = (int) varintOf(tag); + case FIELD_TRAINER_MODEL_TYPE -> model.modelType = (int) varintOf(tag); + case FIELD_TRAINER_TREAT_WHITESPACE_AS_SUFFIX -> + model.treatWhitespaceAsSuffix = varintOf(tag) != 0; + case FIELD_TRAINER_BYTE_FALLBACK -> model.byteFallback = varintOf(tag) != 0; + case FIELD_TRAINER_UNK_ID -> model.unkId = (int) varintOf(tag); default -> skip(tag); } } @@ -135,10 +174,12 @@ private void normalizerSpec(RawModel model, int end) { while (pos < end) { final long tag = varint(); switch ((int) (tag >>> 3)) { - case 2 -> model.precompiledCharsMap = bytes(lenPayload(tag)); - case 3 -> model.addDummyPrefix = varintOf(tag) != 0; - case 4 -> model.removeExtraWhitespaces = varintOf(tag) != 0; - case 5 -> model.escapeWhitespaces = varintOf(tag) != 0; + case FIELD_NORMALIZER_PRECOMPILED_CHARSMAP -> + model.precompiledCharsMap = bytes(lenPayload(tag)); + case FIELD_NORMALIZER_ADD_DUMMY_PREFIX -> model.addDummyPrefix = varintOf(tag) != 0; + case FIELD_NORMALIZER_REMOVE_EXTRA_WHITESPACES -> + model.removeExtraWhitespaces = varintOf(tag) != 0; + case FIELD_NORMALIZER_ESCAPE_WHITESPACES -> model.escapeWhitespaces = varintOf(tag) != 0; default -> skip(tag); } } @@ -154,15 +195,15 @@ private void normalizerSpec(RawModel model, int end) { private void selfTestData(RawModel model, int end) { while (pos < end) { final long tag = varint(); - if ((int) (tag >>> 3) == 1) { + if ((int) (tag >>> 3) == FIELD_SELF_TEST_SAMPLES) { final int sampleEnd = lenPayload(tag); String input = null; String expected = null; while (pos < sampleEnd) { final long sampleTag = varint(); switch ((int) (sampleTag >>> 3)) { - case 1 -> input = utf8(lenPayload(sampleTag)); - case 2 -> expected = utf8(lenPayload(sampleTag)); + case FIELD_SAMPLE_INPUT -> input = utf8(lenPayload(sampleTag)); + case FIELD_SAMPLE_EXPECTED -> expected = utf8(lenPayload(sampleTag)); default -> skip(sampleTag); } } @@ -196,6 +237,13 @@ private int lenPayload(long tag) { return pos + (int) length; } + /** + * Reads the varint value of a field after checking its wire type. + * + * @param tag The field tag, whose wire type must be varint. + * @return The decoded value. + * @throws IllegalArgumentException Thrown if the wire type is wrong or the varint is malformed. + */ private long varintOf(long tag) { if ((tag & 7) != WIRE_VARINT) { throw malformed("field " + (tag >>> 3) + " is not a varint"); @@ -203,6 +251,13 @@ private long varintOf(long tag) { return varint(); } + /** + * Reads the little-endian 32-bit float value of a field after checking its wire type. + * + * @param tag The field tag, whose wire type must be 32-bit. + * @return The decoded float. + * @throws IllegalArgumentException Thrown if the wire type is wrong or the input is truncated. + */ private float fixed32Float(long tag) { if ((tag & 7) != WIRE_FIXED32) { throw malformed("field " + (tag >>> 3) + " is not a 32-bit value"); @@ -216,12 +271,24 @@ private float fixed32Float(long tag) { return Float.intBitsToFloat(bits); } + /** + * Decodes the bytes from the current position up to {@code end} as UTF-8, advancing past them. + * + * @param end The exclusive end offset of the payload. + * @return The decoded string. + */ private String utf8(int end) { final String s = new String(data, pos, end - pos, StandardCharsets.UTF_8); pos = end; return s; } + /** + * Copies the bytes from the current position up to {@code end}, advancing past them. + * + * @param end The exclusive end offset of the payload. + * @return The copied bytes. + */ private byte[] bytes(int end) { final byte[] b = new byte[end - pos]; System.arraycopy(data, pos, b, 0, b.length); @@ -268,6 +335,12 @@ private void skip(long tag) { } } + /** + * Advances the position by a fixed number of bytes. + * + * @param count The number of bytes to skip. + * @throws IllegalArgumentException Thrown if fewer than {@code count} bytes remain. + */ private void advance(int count) { if (pos + count > data.length) { throw malformed("truncated field"); @@ -275,6 +348,12 @@ private void advance(int count) { pos += count; } + /** + * Creates the exception for malformed input, carrying the current byte position. + * + * @param detail A short description of what is malformed. + * @return The exception to throw. + */ private IllegalArgumentException malformed(String detail) { return new IllegalArgumentException( "The model data is malformed at byte " + pos + ": " + detail + "."); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java index 795a251bc8..5253a94675 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java @@ -16,6 +16,8 @@ */ package opennlp.subword.sentencepiece; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Comparator; @@ -27,7 +29,9 @@ * 256-entry direct table and narrow nodes scan a short sorted label slice; both layouts enumerate * identical transitions.

*/ -final class PieceTrie { +final class PieceTrie implements Serializable { + + private static final long serialVersionUID = 30340094783102906L; /** The node id returned when no transition exists. */ static final int DEAD = -1; @@ -45,6 +49,15 @@ final class PieceTrie { private final int[] directStart; private final int[] directPool; + /** + * Wraps the packed arrays produced by {@link Builder} and derives the direct-dispatch tables + * for wide nodes. + * + * @param childStart Per node, the start of its edge slice; one trailing entry marks the end. + * @param labels The transition label of every edge. + * @param childNodes The target node of every edge, parallel to {@code labels}. + * @param values Per node, the accepted piece id, or {@code -1}. + */ private PieceTrie(int[] childStart, byte[] labels, int[] childNodes, int[] values) { this.childStart = childStart; this.labels = labels; @@ -61,7 +74,7 @@ private PieceTrie(int[] childStart, byte[] labels, int[] childNodes, int[] value } } this.directPool = new int[wide * 256]; - java.util.Arrays.fill(directPool, DEAD); + Arrays.fill(directPool, DEAD); for (int node = 0; node < values.length; node++) { final int direct = directStart[node]; if (direct >= 0) { @@ -131,6 +144,41 @@ int value(int node) { return values[node]; } + /** + * Returns the byte length of the longest piece in this trie that is a prefix of + * {@code input[from, inputLength)}. + * + * @param input The UTF-8 input buffer; must not be null. + * @param inputLength The number of valid bytes in {@code input}. + * @param from The offset to match from. + * @return The matched length in bytes, or zero when no piece matches. + */ + int longestMatch(byte[] input, int inputLength, int from) { + int node = root(); + int longest = 0; + for (int i = from; i < inputLength; i++) { + node = step(node, input[i]); + if (node == DEAD) { + break; + } + if (value(node) >= 0) { + longest = i - from + 1; + } + } + return longest; + } + + /** + * Creates the exception reported wherever a vocabulary piece turns out to be defined twice, + * keeping the message identical across all detection sites. + * + * @param piece The duplicated piece content. + * @return The exception to throw. + */ + static IllegalArgumentException duplicatePiece(String piece) { + return new IllegalArgumentException("The piece '" + piece + "' is defined more than once."); + } + // Builds the packed form from keys sorted by unsigned byte order. Key ranges sharing a prefix // are contiguous after the sort, so each recursion partitions its range by the byte at the // current depth. @@ -150,6 +198,14 @@ private static final class Builder { private int nextNode; private int nextEdge; + /** + * Prepares a builder over the keys and their sort order; {@link #count} and {@link #fill} + * perform the actual construction. + * + * @param pieces The UTF-8 bytes of each piece. + * @param ids The id stored for each piece, parallel to {@code pieces}. + * @param order The indices of {@code pieces} sorted by unsigned byte order. + */ Builder(byte[][] pieces, int[] ids, Integer[] order) { this.pieces = pieces; this.ids = ids; @@ -172,9 +228,7 @@ void count(int from, int to, int depth) { i++; // A second key ending at the same depth is a duplicate; the sort made them adjacent. if (i < to && pieces[order[i]].length == depth) { - throw new IllegalArgumentException("The piece '" - + new String(pieces[order[i]], java.nio.charset.StandardCharsets.UTF_8) - + "' is defined more than once."); + throw duplicatePiece(new String(pieces[order[i]], StandardCharsets.UTF_8)); } } while (i < to) { @@ -213,9 +267,7 @@ int fill(int from, int to, int depth) { int i = from; if (i < to && pieces[order[i]].length == depth) { if (values[node] != -1 || (i + 1 < to && pieces[order[i + 1]].length == depth)) { - throw new IllegalArgumentException( - "The piece '" + new String(pieces[order[i]], java.nio.charset.StandardCharsets.UTF_8) - + "' is defined more than once."); + throw duplicatePiece(new String(pieces[order[i]], StandardCharsets.UTF_8)); } values[node] = ids[order[i]]; i++; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java index 4d5589514e..0dd83c2784 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -16,6 +16,8 @@ */ package opennlp.subword.sentencepiece; +import java.io.Serializable; + /** * The model-embedded text normalizer of a SentencePiece model, operating in UTF-8 byte space. * @@ -26,7 +28,9 @@ * was derived from, with one trailing entry for the end position; that map is what lets every * downstream piece report an exact span of the caller's text.

*/ -final class SentencePieceNormalizer { +final class SentencePieceNormalizer implements Serializable { + + private static final long serialVersionUID = -3059745470932191300L; // U+2581 LOWER ONE EIGHTH BLOCK in UTF-8, the escaped form of a space. static final byte[] SPACE_SYMBOL = {(byte) 0xE2, (byte) 0x96, (byte) 0x81}; @@ -238,6 +242,15 @@ Normalized normalize(byte[] input, int inputLength) { private static final byte[] SINGLE_SPACE = {' '}; + /** + * Appends the space symbol to the normalized output, mapping each of its bytes to the same + * original-byte offset. + * + * @param normalized The normalized-byte builder to append to. + * @param normToOrig The offset-map builder to append to. + * @param spaceSymbol The bytes of the (possibly escaped) space symbol. + * @param consumed The original-byte offset the symbol maps back to. + */ private static void appendSpace(ByteBuilder normalized, IntBuilder normToOrig, byte[] spaceSymbol, int consumed) { normalized.append(spaceSymbol, 0, spaceSymbol.length); @@ -259,7 +272,7 @@ private static void appendSpace(ByteBuilder normalized, IntBuilder normToOrig, */ private void normalizePrefix(byte[] input, int inputLength, int from, Chunk chunk) { if (userDefinedMatcher != null) { - final int matched = longestUserDefinedMatch(input, inputLength, from); + final int matched = userDefinedMatcher.longestMatch(input, inputLength, from); if (matched > 0) { chunk.data = input; chunk.from = from; @@ -303,30 +316,6 @@ private void normalizePrefix(byte[] input, int inputLength, int from, Chunk chun chunk.consumed = charLength; } - /** - * Returns the byte length of the longest user-defined symbol that is a prefix of - * {@code input[from, inputLength)}. - * - * @param input The UTF-8 input buffer. - * @param inputLength The number of valid bytes in {@code input}. - * @param from The offset to match from. - * @return The matched length in bytes, or zero when no user-defined symbol matches. - */ - private int longestUserDefinedMatch(byte[] input, int inputLength, int from) { - int node = userDefinedMatcher.root(); - int longest = 0; - for (int i = from; i < inputLength; i++) { - node = userDefinedMatcher.step(node, input[i]); - if (node == PieceTrie.DEAD) { - break; - } - if (userDefinedMatcher.value(node) >= 0) { - longest = i - from + 1; - } - } - return longest; - } - /** * Returns the byte length of a UTF-8 sequence from its lead byte; trail and malformed lead bytes * report one byte. diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 961bb58b3c..e8c0e04ee7 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -22,9 +22,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.IntUnaryOperator; import opennlp.tools.tokenize.SubwordPiece; import opennlp.tools.tokenize.SubwordTokenizer; @@ -43,6 +45,11 @@ * for reuse outside tokenization.

* *

Instances are immutable after loading and safe for concurrent use by multiple threads.

+ * + * @see SentencePiece + * @see Kudo & Richardson (EMNLP 2018), + * "SentencePiece: A simple and language independent subword tokenizer and detokenizer for + * Neural Text Processing" */ public final class SentencePieceTokenizer implements SubwordTokenizer, OffsetAwareNormalizer { @@ -82,6 +89,13 @@ public enum Algorithm { private final List selfTestInputs; private final List selfTestExpected; + /** + * Validates a parsed model and derives the runtime structures: the piece maps, the byte-piece + * table, the normalizer, and the encoder matching the model's algorithm. + * + * @param model The parsed model description. + * @throws IllegalArgumentException Thrown if the model is structurally invalid. + */ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { final int count = model.pieces.size(); pieces = model.pieces.toArray(new String[0]); @@ -106,7 +120,7 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { reservedPieces = new HashMap<>(); final List userDefined = new ArrayList<>(); byteToId = new int[256]; - java.util.Arrays.fill(byteToId, -1); + Arrays.fill(byteToId, -1); int foundUnkId = -1; float minScore = Float.MAX_VALUE; for (int i = 0; i < count; i++) { @@ -124,7 +138,7 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { final Map target = isMain || algorithm == Algorithm.BPE ? mainPieces : reservedPieces; if (mainPieces.containsKey(piece) || reservedPieces.containsKey(piece)) { - throw new IllegalArgumentException("The piece '" + piece + "' is defined more than once."); + throw PieceTrie.duplicatePiece(piece); } target.put(piece, i); switch (types[i]) { @@ -165,7 +179,8 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { } } - final PieceTrie userDefinedMatcher = userDefined.isEmpty() ? null : trieOf(userDefined, id -> 0); + final PieceTrie userDefinedMatcher = + userDefined.isEmpty() ? null : trieOf(userDefined, id -> 0); normalizer = new SentencePieceNormalizer(model.precompiledCharsMap, model.addDummyPrefix, model.removeExtraWhitespaces, model.escapeWhitespaces, model.treatWhitespaceAsSuffix, @@ -204,8 +219,14 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { selfTestExpected = List.copyOf(model.selfTestExpected); } - private static PieceTrie trieOf(List pieceList, - java.util.function.IntUnaryOperator idOf) { + /** + * Builds a {@link PieceTrie} over the given pieces. + * + * @param pieceList The pieces to index. + * @param idOf Maps a piece's index in {@code pieceList} to the id the trie stores for it. + * @return The packed trie. + */ + private static PieceTrie trieOf(List pieceList, IntUnaryOperator idOf) { final byte[][] keys = new byte[pieceList.size()][]; final int[] ids = new int[pieceList.size()]; for (int i = 0; i < keys.length; i++) { @@ -246,11 +267,7 @@ public static SentencePieceTokenizer load(InputStream in) throws IOException { return new SentencePieceTokenizer(ModelProtoReader.read(in.readAllBytes())); } - /** - * {@inheritDoc} - * - * @throws IllegalArgumentException Thrown if {@code text} is null. - */ + /** {@inheritDoc} */ @Override public List encode(CharSequence text) { if (text == null) { @@ -526,13 +543,16 @@ List selfTestExpected() { return selfTestExpected; } + // The prefix of a byte-fallback piece string; a full piece has the form "<0xAB>". + private static final String BYTE_PIECE_PREFIX = "<0x"; + // "<0xAB>" piece strings for all byte values, as byte fallback emits them. private static final String[] BYTE_PIECES = new String[256]; static { final char[] hex = "0123456789ABCDEF".toCharArray(); for (int b = 0; b < 256; b++) { - BYTE_PIECES[b] = "<0x" + hex[b >>> 4] + hex[b & 0xF] + ">"; + BYTE_PIECES[b] = BYTE_PIECE_PREFIX + hex[b >>> 4] + hex[b & 0xF] + ">"; } } @@ -543,7 +563,7 @@ List selfTestExpected() { * @return The byte value in {@code [0, 255]}, or {@code -1} when the string is not a byte piece. */ private static int parseBytePiece(String piece) { - if (piece.length() != 6 || !piece.startsWith("<0x") || piece.charAt(5) != '>') { + if (piece.length() != 6 || !piece.startsWith(BYTE_PIECE_PREFIX) || piece.charAt(5) != '>') { return -1; } final int high = Character.digit(piece.charAt(3), 16); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java index c87662bef2..1efa00310d 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java @@ -16,6 +16,7 @@ */ package opennlp.subword.sentencepiece; +import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -27,9 +28,14 @@ *

Characters no piece covers fall back to the unknown id with a fixed penalty below the lowest * piece score, and user-defined symbols receive a length-based bonus score so they always win.

*/ -final class UnigramEncoder { +final class UnigramEncoder implements Serializable { + + private static final long serialVersionUID = 5648005733414803707L; private static final float UNK_PENALTY = 10.0f; + // The score of a user-defined symbol is this bonus per matched byte beyond the first instead + // of a trained log-probability, so longer user-defined matches always win the best path. + private static final float USER_DEFINED_LENGTH_BONUS = 0.1f; private static final float SCORE_RESET_THRESHOLD = 100000.0f; private final PieceTrie trie; @@ -65,8 +71,12 @@ final class UnigramEncoder { * @param normalized The buffer holding the normalized UTF-8 bytes; must not be null. * @param size The number of valid bytes in {@code normalized}. * @return The best-path segments covering all bytes, in text order. + * @throws IllegalArgumentException Thrown if {@code normalized} is null. */ List encode(byte[] normalized, int size) { + if (normalized == null) { + throw new IllegalArgumentException("The normalized buffer must not be null."); + } if (size == 0) { return List.of(); } @@ -118,7 +128,8 @@ List encode(byte[] normalized, int size) { maxFrontier = Math.max(maxFrontier, keyPos); final int length = keyPos - startsAt; // User-defined symbols receive a length bonus instead of a trained score. - final float score = userDefined[id] ? 0.1f * (length - 1) : scores[id]; + final float score = userDefined[id] + ? USER_DEFINED_LENGTH_BONUS * (length - 1) : scores[id]; final float candidate = score + bestScoreTillHere; final int slot = 3 * keyPos; if (best[slot] == -1 || candidate > Float.intBitsToFloat(best[slot + 1])) { diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java index e830816e64..4f2bd95f0f 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java @@ -32,6 +32,14 @@ final class Utf8Text { private final int[] byteToChar; private final int charLength; + /** + * Wraps an encoded buffer and its offset map. + * + * @param bytes The UTF-8 buffer. + * @param byteLength The number of valid bytes in {@code bytes}. + * @param byteToChar The byte-to-UTF-16 offset map, or null for pure-ASCII text. + * @param charLength The length of the original text in UTF-16 units. + */ private Utf8Text(byte[] bytes, int byteLength, int[] byteToChar, int charLength) { this.bytes = bytes; this.byteLength = byteLength; diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java new file mode 100644 index 0000000000..11e54d2bef --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import opennlp.tools.tokenize.SubwordPiece; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Shared support for the tab-separated parity fixture files produced by the + * {@code gen_fixtures.py} and {@code gen_real_fixtures.py} scripts in the test resources: one + * line per input, holding the input, the expected piece count, four columns per expected piece + * (content, id, start, end), and the expected normalized form. + */ +final class SentencePieceFixtures { + + private SentencePieceFixtures() { + } + + /** + * One parsed fixture line: an input with the piece sequence and normalized form the reference + * implementation produced for it. + * + * @param input The text to encode. + * @param pieces The expected pieces with ids and original-text spans, in text order. + * @param normalized The expected normalized form of {@code input}. + */ + record Fixture(String input, List pieces, String normalized) { + } + + /** + * Reads all fixture lines from a reader. + * + * @param reader The reader positioned at the start of a fixture file; must not be null. + * @return The parsed fixtures in file order. + * @throws IOException Thrown if the reader fails. + */ + static List read(BufferedReader reader) throws IOException { + final List fixtures = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + final String[] cols = line.split("\t", -1); + final String input = unescape(cols[0]); + final int count = Integer.parseInt(cols[1]); + final List pieces = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + pieces.add(new SubwordPiece(unescape(cols[2 + i * 4]), + Integer.parseInt(cols[3 + i * 4]), Integer.parseInt(cols[4 + i * 4]), + Integer.parseInt(cols[5 + i * 4]))); + } + fixtures.add(new Fixture(input, pieces, unescape(cols[2 + count * 4]))); + } + return fixtures; + } + + /** + * Asserts that a tokenizer reproduces one fixture exactly: the piece sequence with ids and + * spans, and the normalized form. + * + * @param tokenizer The tokenizer under test. + * @param fixture The expected encoding. + * @param context A prefix for failure messages that identifies the model and input. + */ + static void assertFixture(SentencePieceTokenizer tokenizer, Fixture fixture, String context) { + final List actual = tokenizer.encode(fixture.input()); + assertEquals(fixture.pieces().size(), actual.size(), + context + " piece count; got " + actual); + for (int i = 0; i < actual.size(); i++) { + final SubwordPiece expected = fixture.pieces().get(i); + final SubwordPiece got = actual.get(i); + assertEquals(expected.piece(), got.piece(), context + " piece " + i); + assertEquals(expected.id(), got.id(), context + " id of piece " + i); + assertEquals(expected.start(), got.start(), context + " start of piece " + i); + assertEquals(expected.end(), got.end(), context + " end of piece " + i); + } + assertEquals(fixture.normalized(), tokenizer.normalize(fixture.input()).toString(), + context + " normalized form"); + } + + /** + * Reverses the fixture files' escaping of tab, newline, carriage return, and backslash. + * + * @param s The escaped column content. + * @return The unescaped text. + * @throws IllegalArgumentException Thrown if an unknown escape sequence occurs. + */ + static String unescape(String s) { + final StringBuilder out = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + if (c == '\\' && i + 1 < s.length()) { + i++; + switch (s.charAt(i)) { + case 't' -> out.append('\t'); + case 'n' -> out.append('\n'); + case 'r' -> out.append('\r'); + case '\\' -> out.append('\\'); + default -> throw new IllegalArgumentException("bad escape in fixture: " + s); + } + } else { + out.append(c); + } + } + return out.toString(); + } +} diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java index 66595eff8b..a65df08991 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java @@ -21,7 +21,9 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -45,7 +47,7 @@ class SentencePieceModelValidationTest { @Test void testNullAndEmptyInputFailLoudly() { assertThrows(IllegalArgumentException.class, - () -> SentencePieceTokenizer.load((java.nio.file.Path) null)); + () -> SentencePieceTokenizer.load((Path) null)); assertThrows(IllegalArgumentException.class, () -> SentencePieceTokenizer.load((InputStream) null)); assertThrows(IllegalArgumentException.class, @@ -62,7 +64,7 @@ void testGarbageBytesFailLoudly() { @Test void testTruncatedModelFailsLoudly() throws IOException { final byte[] whole = readModel(); - final byte[] truncated = java.util.Arrays.copyOf(whole, whole.length / 3); + final byte[] truncated = Arrays.copyOf(whole, whole.length / 3); assertThrows(IllegalArgumentException.class, () -> SentencePieceTokenizer.load(new ByteArrayInputStream(truncated))); } diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java index 7d970e3c5e..95730ef911 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java @@ -21,7 +21,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.StringJoiner; @@ -30,8 +29,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import opennlp.tools.tokenize.SubwordPiece; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -63,22 +60,10 @@ static SentencePieceTokenizer tokenizer(String model) { void testFixtureParity(String model) throws IOException { final SentencePieceTokenizer tokenizer = tokenizer(model); int lines = 0; - for (final Fixture fixture : fixtures(model)) { + for (final SentencePieceFixtures.Fixture fixture : fixtures(model)) { lines++; - final List actual = tokenizer.encode(fixture.input); - final String context = model + " input <" + fixture.input + ">"; - assertEquals(fixture.pieces.size(), actual.size(), - context + " piece count; got " + actual); - for (int i = 0; i < actual.size(); i++) { - final SubwordPiece expected = fixture.pieces.get(i); - final SubwordPiece got = actual.get(i); - assertEquals(expected.piece(), got.piece(), context + " piece " + i); - assertEquals(expected.id(), got.id(), context + " id of piece " + i); - assertEquals(expected.start(), got.start(), context + " start of piece " + i); - assertEquals(expected.end(), got.end(), context + " end of piece " + i); - } - assertEquals(fixture.normalized, tokenizer.normalize(fixture.input).toString(), - context + " normalized form"); + SentencePieceFixtures.assertFixture(tokenizer, fixture, + model + " input <" + fixture.input() + ">"); } assertTrue(lines >= 30, "the fixture file must not be empty or truncated"); } @@ -101,50 +86,12 @@ void testEmbeddedSelfTestSamples(String model) { } } - private record Fixture(String input, List pieces, String normalized) { - } - - private static List fixtures(String model) throws IOException { - final List fixtures = new ArrayList<>(); + private static List fixtures(String model) throws IOException { try (InputStream in = SentencePieceParityTest.class.getResourceAsStream(model + ".fixtures.tsv")) { assertNotNull(in, "missing test resource " + model + ".fixtures.tsv"); - final BufferedReader reader = - new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); - String line; - while ((line = reader.readLine()) != null) { - final String[] cols = line.split("\t", -1); - final String input = unescape(cols[0]); - final int count = Integer.parseInt(cols[1]); - final List pieces = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - pieces.add(new SubwordPiece(unescape(cols[2 + i * 4]), - Integer.parseInt(cols[3 + i * 4]), Integer.parseInt(cols[4 + i * 4]), - Integer.parseInt(cols[5 + i * 4]))); - } - fixtures.add(new Fixture(input, pieces, unescape(cols[2 + count * 4]))); - } - } - return fixtures; - } - - private static String unescape(String s) { - final StringBuilder out = new StringBuilder(s.length()); - for (int i = 0; i < s.length(); i++) { - final char c = s.charAt(i); - if (c == '\\' && i + 1 < s.length()) { - i++; - switch (s.charAt(i)) { - case 't' -> out.append('\t'); - case 'n' -> out.append('\n'); - case 'r' -> out.append('\r'); - case '\\' -> out.append('\\'); - default -> throw new IllegalArgumentException("bad escape in fixture: " + s); - } - } else { - out.append(c); - } + return SentencePieceFixtures.read( + new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))); } - return out.toString(); } } diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java index 1255827ba1..fd1a9ca9ef 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java @@ -26,9 +26,6 @@ import org.junit.jupiter.api.Test; -import opennlp.tools.tokenize.SubwordPiece; - -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -64,49 +61,14 @@ void testRealModelParity() throws IOException { private static void assertModel(Path modelPath, Path fixturesPath) throws IOException { final SentencePieceTokenizer tokenizer = SentencePieceTokenizer.load(modelPath); - int lines = 0; + final List fixtures; try (BufferedReader reader = Files.newBufferedReader(fixturesPath, StandardCharsets.UTF_8)) { - String line; - while ((line = reader.readLine()) != null) { - lines++; - final String[] cols = line.split("\t", -1); - final String input = unescape(cols[0]); - final int count = Integer.parseInt(cols[1]); - final String context = modelPath.getFileName() + " input <" + input + ">"; - - final List actual = tokenizer.encode(input); - assertEquals(count, actual.size(), context + " piece count; got " + actual); - for (int i = 0; i < count; i++) { - final SubwordPiece got = actual.get(i); - assertEquals(unescape(cols[2 + i * 4]), got.piece(), context + " piece " + i); - assertEquals(Integer.parseInt(cols[3 + i * 4]), got.id(), context + " id " + i); - assertEquals(Integer.parseInt(cols[4 + i * 4]), got.start(), context + " start " + i); - assertEquals(Integer.parseInt(cols[5 + i * 4]), got.end(), context + " end " + i); - } - assertEquals(unescape(cols[2 + count * 4]), tokenizer.normalize(input).toString(), - context + " normalized form"); - } + fixtures = SentencePieceFixtures.read(reader); } - assertTrue(lines >= 30, modelPath.getFileName() + " fixtures must not be truncated"); - } - - private static String unescape(String s) { - final StringBuilder out = new StringBuilder(s.length()); - for (int i = 0; i < s.length(); i++) { - final char c = s.charAt(i); - if (c == '\\' && i + 1 < s.length()) { - i++; - switch (s.charAt(i)) { - case 't' -> out.append('\t'); - case 'n' -> out.append('\n'); - case 'r' -> out.append('\r'); - case '\\' -> out.append('\\'); - default -> throw new IllegalArgumentException("bad escape in fixture: " + s); - } - } else { - out.append(c); - } + for (final SentencePieceFixtures.Fixture fixture : fixtures) { + SentencePieceFixtures.assertFixture(tokenizer, fixture, + modelPath.getFileName() + " input <" + fixture.input() + ">"); } - return out.toString(); + assertTrue(fixtures.size() >= 30, modelPath.getFileName() + " fixtures must not be truncated"); } } diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java new file mode 100644 index 0000000000..f1b56b7697 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; + +/** + * Asserts the {@code Serializable} contract inherited through + * {@code opennlp.tools.util.normalizer.CharSequenceNormalizer}: a tokenizer round-tripped + * through Java object serialization must encode and normalize exactly like the original. + */ +class SentencePieceTokenizerSerializationTest { + + private static final String[] INPUTS = { + "", + "The quick brown fox jumps over the lazy dog.", + " Hello world ", + "tokenization and segmentation", + "caf\u00e9 na\u00efve \u4e2d\u6587" + }; + + @ParameterizedTest + @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", + "tiny-unigram-identity", "tiny-unigram-suffix"}) + void testRoundTripPreservesEncoding(String model) throws IOException, ClassNotFoundException { + final SentencePieceTokenizer original = SentencePieceParityTest.tokenizer(model); + + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(original); + } + final SentencePieceTokenizer copy; + try (ObjectInputStream in = + new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + copy = (SentencePieceTokenizer) in.readObject(); + } + + assertEquals(original.algorithm(), copy.algorithm(), model + " algorithm"); + assertEquals(original.vocabularySize(), copy.vocabularySize(), model + " vocabulary size"); + for (final String input : INPUTS) { + final String context = model + " input <" + input + ">"; + assertIterableEquals(original.encode(input), copy.encode(input), context + " pieces"); + assertEquals(original.normalize(input).toString(), copy.normalize(input).toString(), + context + " normalized form"); + } + } +} From e43010913354048059a7a6b5dee2e8e56e775f2f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 17 Jul 2026 08:14:44 -0400 Subject: [PATCH 12/82] OPENNLP-1885: Guard tokenizer deserialization with an allow-listing ObjectInputFilter SentencePieceTokenizer gains serialize(OutputStream) and deserialize(InputStream) methods. Reads are filtered through an ObjectInputFilter that allow-lists only the classes reachable from a legitimate tokenizer graph and bounds graph depth, references, and array length; foreign payloads are rejected with InvalidClassException before being materialised. Limits are adjustable through a DeserializationLimits record for unusually large vocabularies; the allow-list is not configurable. The serialVersionUID is recomputed for the new public methods. --- .../sentencepiece/SentencePieceTokenizer.java | 201 +++++++++++++++++- ...ntencePieceTokenizerSerializationTest.java | 86 ++++++++ 2 files changed, 286 insertions(+), 1 deletion(-) diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index e8c0e04ee7..4d73f54016 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -18,6 +18,10 @@ import java.io.IOException; import java.io.InputStream; +import java.io.ObjectInputFilter; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -26,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.IntUnaryOperator; import opennlp.tools.tokenize.SubwordPiece; @@ -46,6 +51,12 @@ * *

Instances are immutable after loading and safe for concurrent use by multiple threads.

* + *

Beyond {@link #load(Path) loading} the native {@code .model} format, a tokenizer can be + * persisted with {@link #serialize(OutputStream)} and read back with + * {@link #deserialize(InputStream)}. Reads are guarded by an {@link java.io.ObjectInputFilter} + * that allow-lists only the classes reachable from a legitimate tokenizer graph and bounds graph + * depth, references, and array length.

+ * * @see SentencePiece * @see Kudo & Richardson (EMNLP 2018), * "SentencePiece: A simple and language independent subword tokenizer and detokenizer for @@ -54,7 +65,7 @@ public final class SentencePieceTokenizer implements SubwordTokenizer, OffsetAwareNormalizer { // Serializable through the OffsetAwareNormalizer contract. - private static final long serialVersionUID = -7114394869301531147L; + private static final long serialVersionUID = -4472058014098085134L; /** The segmentation algorithm a model was trained with. */ public enum Algorithm { @@ -267,6 +278,194 @@ public static SentencePieceTokenizer load(InputStream in) throws IOException { return new SentencePieceTokenizer(ModelProtoReader.read(in.readAllBytes())); } + /** + * Serializes this tokenizer to the given {@link OutputStream} using Java object serialization. + * The resulting stream can be read back with {@link #deserialize(InputStream)}. + * + * @param out The {@link OutputStream} to write to; must not be null. + * @throws IOException Thrown if IO errors occurred during serialization. + * @throws IllegalArgumentException Thrown if {@code out} is null. + */ + public void serialize(OutputStream out) throws IOException { + if (out == null) { + throw new IllegalArgumentException("The output stream must not be null."); + } + try (ObjectOutputStream oos = new ObjectOutputStream(out)) { + oos.writeObject(this); + } + } + + /** + * Deserializes a {@link SentencePieceTokenizer} from the given {@link InputStream} using + * {@link DeserializationLimits#DEFAULT default} resource limits. + * + *

The stream is filtered via an {@link ObjectInputFilter} that allow-lists only the classes + * required to reconstruct a {@link SentencePieceTokenizer}, plus resource limits on graph depth, + * references, and array length. Foreign payloads are rejected with + * {@link java.io.InvalidClassException} before {@link ObjectInputStream#readObject()} + * returns.

+ * + *

Callers should still treat this method as defense-in-depth: only invoke it on streams from + * trusted sources. If the default limits are too tight for an unusually large model, use + * {@link #deserialize(InputStream, DeserializationLimits)} to supply higher limits. The class + * allow-list is intentionally not configurable; loosening it would defeat the purpose of the + * filter.

+ * + * @param in The {@link InputStream} to read from; must not be null. + * @return The reconstructed tokenizer. + * @throws IOException Thrown if IO errors occurred during deserialization, including + * {@link java.io.InvalidClassException} when the stream contains a class outside the + * allow-list or exceeds a resource limit. + * @throws ClassNotFoundException Thrown if required classes are not found. + * @throws IllegalArgumentException Thrown if {@code in} is null. + */ + public static SentencePieceTokenizer deserialize(InputStream in) + throws IOException, ClassNotFoundException { + return deserialize(in, DeserializationLimits.DEFAULT); + } + + /** + * Deserializes a {@link SentencePieceTokenizer} from the given {@link InputStream} using the + * supplied {@link DeserializationLimits resource limits}. + * + *

Use this overload when the {@link DeserializationLimits#DEFAULT default limits} reject a + * legitimate model, for example one with a very large vocabulary. The class allow-list applied + * to the stream is the same as for {@link #deserialize(InputStream)}; only the numeric limits + * change.

+ * + * @param in The {@link InputStream} to read from; must not be null. + * @param limits The {@link DeserializationLimits} to apply; must not be null. + * @return The reconstructed tokenizer. + * @throws IOException Thrown if IO errors occurred during deserialization, including + * {@link java.io.InvalidClassException} when the stream contains a class outside the + * allow-list or exceeds one of the supplied limits. + * @throws ClassNotFoundException Thrown if required classes are not found. + * @throws IllegalArgumentException Thrown if {@code in} or {@code limits} is null. + */ + public static SentencePieceTokenizer deserialize(InputStream in, DeserializationLimits limits) + throws IOException, ClassNotFoundException { + if (in == null) { + throw new IllegalArgumentException("The input stream must not be null."); + } + if (limits == null) { + throw new IllegalArgumentException("The limits must not be null."); + } + try (ObjectInputStream ois = new ObjectInputStream(in)) { + ois.setObjectInputFilter(buildFilter(limits)); + return (SentencePieceTokenizer) ois.readObject(); + } + } + + /** + * Resource limits applied by the {@link ObjectInputFilter} used by + * {@link SentencePieceTokenizer#deserialize(InputStream, DeserializationLimits)}. + * + *

The limits bound graph traversal regardless of the class allow-list and provide + * defense-in-depth against pathological streams. The {@linkplain #DEFAULT default values} are + * generous enough for typical production models; raise them only if a legitimate model is + * rejected.

+ * + * @param maxDepth Maximum object-graph nesting depth. Must be {@code > 0}. + * @param maxRefs Maximum number of internal references the stream may create. + * Must be {@code > 0}. + * @param maxArrayLength Maximum length of any single array allocation requested by the stream. + * Must be {@code > 0}. + */ + public record DeserializationLimits(long maxDepth, long maxRefs, long maxArrayLength) { + + /** + * Default limits. Sized so that models with vocabularies of several hundred thousand pieces + * round-trip while pathological streams stay bounded. + */ + public static final DeserializationLimits DEFAULT = + new DeserializationLimits(MAX_DEPTH_DEFAULT, MAX_REFS_DEFAULT, MAX_ARRAY_DEFAULT); + + /** + * Validates the limits. + * + * @throws IllegalArgumentException Thrown if any of {@code maxDepth}, {@code maxRefs}, or + * {@code maxArrayLength} is {@code <= 0}. + */ + public DeserializationLimits { + if (maxDepth <= 0) { + throw new IllegalArgumentException("maxDepth must be > 0"); + } + if (maxRefs <= 0) { + throw new IllegalArgumentException("maxRefs must be > 0"); + } + if (maxArrayLength <= 0) { + throw new IllegalArgumentException("maxArrayLength must be > 0"); + } + } + } + + private static final long MAX_DEPTH_DEFAULT = 64; + private static final long MAX_REFS_DEFAULT = 5_000_000; + private static final long MAX_ARRAY_DEFAULT = 10_000_000; + + // Allow-list of fully qualified class names that may appear in the serialized graph of a + // SentencePieceTokenizer. Anything else is rejected. + private static final Set ALLOWED_CLASSES = Set.of( + "opennlp.subword.sentencepiece.SentencePieceTokenizer", + "opennlp.subword.sentencepiece.SentencePieceTokenizer$Algorithm", + "opennlp.subword.sentencepiece.SentencePieceNormalizer", + "opennlp.subword.sentencepiece.UnigramEncoder", + "opennlp.subword.sentencepiece.BpeEncoder", + "opennlp.subword.sentencepiece.PieceTrie", + "opennlp.subword.sentencepiece.DoubleArrayTrie", + // JDK types used in field declarations. ObjectInputStream invokes the filter for every + // class descriptor in the inheritance chain, not only for the runtime class - so the + // abstract superclasses java.lang.Number (super of Integer) and java.lang.Enum (super of + // Algorithm) must be allow-listed even though no instance of either appears in the stream. + "java.lang.String", + "java.lang.Number", + "java.lang.Integer", + "java.lang.Enum", + "java.util.HashMap", + // HashMap.readObject() requests permission to allocate a Map.Entry[] before reading + // entries; the array type itself never appears as a value in the stream. + "java.util.Map$Entry", + // The unmodifiable lists created by List.copyOf serialize through the CollSer proxy, + // which requests an Object[] allocation for the elements, and the filter is also invoked + // for the concrete list class the proxy resolves to. + "java.util.CollSer", + "java.util.ImmutableCollections$List12", + "java.util.ImmutableCollections$ListN", + "java.lang.Object" + ); + + /** + * Builds the {@link ObjectInputFilter} enforcing the class allow-list and the given limits. + * + * @param limits The resource limits to enforce; never null here. + * @return The filter to install on the reading {@link ObjectInputStream}. + */ + private static ObjectInputFilter buildFilter(DeserializationLimits limits) { + return info -> { + if (info.depth() > limits.maxDepth() + || info.references() > limits.maxRefs() + || info.arrayLength() > limits.maxArrayLength()) { + return ObjectInputFilter.Status.REJECTED; + } + + final Class serialClass = info.serialClass(); + if (serialClass == null) { + return ObjectInputFilter.Status.UNDECIDED; + } + + Class componentType = serialClass; + while (componentType.isArray()) { + componentType = componentType.getComponentType(); + } + if (componentType.isPrimitive()) { + return ObjectInputFilter.Status.ALLOWED; + } + return ALLOWED_CLASSES.contains(componentType.getName()) + ? ObjectInputFilter.Status.ALLOWED + : ObjectInputFilter.Status.REJECTED; + }; + } + /** {@inheritDoc} */ @Override public List encode(CharSequence text) { diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java index f1b56b7697..c90e7201ac 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java @@ -19,19 +19,27 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InvalidClassException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.util.ArrayList; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Asserts the {@code Serializable} contract inherited through * {@code opennlp.tools.util.normalizer.CharSequenceNormalizer}: a tokenizer round-tripped * through Java object serialization must encode and normalize exactly like the original. + * Also asserts the guarded read path of + * {@link SentencePieceTokenizer#deserialize(InputStream)}: foreign payloads and streams + * exceeding the resource limits are rejected before materialisation. */ class SentencePieceTokenizerSerializationTest { @@ -68,4 +76,82 @@ void testRoundTripPreservesEncoding(String model) throws IOException, ClassNotFo context + " normalized form"); } } + + /** + * Serializes the tokenizer of the given fixture model through + * {@link SentencePieceTokenizer#serialize(OutputStream)}. + * + * @param model The fixture model name. + * @return The serialized bytes. + */ + private static byte[] serialized(String model) throws IOException { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + SentencePieceParityTest.tokenizer(model).serialize(bytes); + return bytes.toByteArray(); + } + + @ParameterizedTest + @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", + "tiny-unigram-identity", "tiny-unigram-suffix"}) + void testGuardedDeserializePreservesEncoding(String model) + throws IOException, ClassNotFoundException { + final SentencePieceTokenizer original = SentencePieceParityTest.tokenizer(model); + final SentencePieceTokenizer copy = + SentencePieceTokenizer.deserialize(new ByteArrayInputStream(serialized(model))); + + assertEquals(original.algorithm(), copy.algorithm(), model + " algorithm"); + for (final String input : INPUTS) { + final String context = model + " input <" + input + ">"; + assertIterableEquals(original.encode(input), copy.encode(input), context + " pieces"); + } + } + + /** + * Verifies that a stream whose top-level object is not on the allow-list is rejected + * before it is materialised, even though its classes are harmless JDK types. + */ + @Test + void testForeignPayloadIsRejected() throws IOException { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + final ArrayList foreign = new ArrayList<>(); + foreign.add("not a tokenizer"); + out.writeObject(foreign); + } + assertThrows(InvalidClassException.class, () -> + SentencePieceTokenizer.deserialize(new ByteArrayInputStream(bytes.toByteArray()))); + } + + /** + * Verifies that a legitimate stream is rejected when it exceeds the supplied resource + * limits, so the limits bound the graph regardless of the class allow-list. + */ + @Test + void testStreamExceedingLimitsIsRejected() throws IOException { + final byte[] legitimate = serialized("tiny-unigram"); + final SentencePieceTokenizer.DeserializationLimits tight = + new SentencePieceTokenizer.DeserializationLimits(1, 1, 1); + assertThrows(InvalidClassException.class, () -> + SentencePieceTokenizer.deserialize(new ByteArrayInputStream(legitimate), tight)); + } + + /** + * Verifies that null arguments are rejected with {@link IllegalArgumentException} at the + * API boundary. + */ + @Test + void testNullArgumentsAreRejected() throws IOException { + final SentencePieceTokenizer tokenizer = SentencePieceParityTest.tokenizer("tiny-unigram"); + assertThrows(IllegalArgumentException.class, () -> tokenizer.serialize(null)); + assertThrows(IllegalArgumentException.class, () -> + SentencePieceTokenizer.deserialize(null)); + assertThrows(IllegalArgumentException.class, () -> + SentencePieceTokenizer.deserialize(new ByteArrayInputStream(new byte[0]), null)); + assertThrows(IllegalArgumentException.class, () -> + new SentencePieceTokenizer.DeserializationLimits(0, 1, 1)); + assertThrows(IllegalArgumentException.class, () -> + new SentencePieceTokenizer.DeserializationLimits(1, 0, 1)); + assertThrows(IllegalArgumentException.class, () -> + new SentencePieceTokenizer.DeserializationLimits(1, 1, 0)); + } } From b843cf7baba951bfe21683730939a33ff66d79e9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:40:02 -0400 Subject: [PATCH 13/82] OPENNLP-1885: Cite the SentencePiece usage example test in the manual Add SentencePieceUsageExampleTest asserting the load-and-encode workflow and point the tokenizer manual section at it. --- opennlp-docs/src/docbkx/tokenizer.xml | 9 +-- .../SentencePieceUsageExampleTest.java | 69 +++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index b6b27c5c16..fa7a509f8e 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -574,10 +574,11 @@ for (SubwordPiece piece : tokenizer.encode("Subword pieces keep their original o int[] ids = tokenizer.encodeToIds("Ready for the embedding layer.");]]> - The vocabulary can be inspected through vocabularySize, - idToPiece, pieceToId, and score, and the - algorithm method reports whether the model uses the unigram or the - byte-pair encoding algorithm. + SentencePieceUsageExampleTest asserts the load-and-encode workflow + shown here. The vocabulary can be inspected through + vocabularySize, idToPiece, pieceToId, and + score, and the algorithm method reports whether the + model uses the unigram or the byte-pair encoding algorithm. The model's own normalizer is also exposed directly: diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java new file mode 100644 index 0000000000..33c375574b --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.subword.sentencepiece; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.tokenize.SubwordPiece; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins the cookbook path documented in {@code tokenizer.xml}: load a + * {@link SentencePieceTokenizer} from a {@code .model} file, encode text to pieces with + * original offsets, and obtain id arrays. + */ +public class SentencePieceUsageExampleTest { + + @Test + void testLoadEncodeAndEncodeToIds(@TempDir Path dir) throws IOException { + final Path modelFile = dir.resolve("spiece.model"); + try (InputStream in = SentencePieceUsageExampleTest.class + .getResourceAsStream("tiny-unigram.model")) { + assertNotNull(in, "missing test resource tiny-unigram.model"); + Files.copy(in, modelFile); + } + + final SentencePieceTokenizer tokenizer = SentencePieceTokenizer.load(modelFile); + final String text = "hello world"; + final List pieces = tokenizer.encode(text); + assertFalse(pieces.isEmpty()); + for (final SubwordPiece piece : pieces) { + assertTrue(piece.id() >= 0); + assertTrue(piece.start() >= 0); + assertTrue(piece.end() <= text.length()); + // Control or whitespace pieces may report an empty span (start == end). + assertTrue(piece.start() <= piece.end()); + } + + final int[] ids = tokenizer.encodeToIds(text); + assertEquals(pieces.size(), ids.length); + for (int i = 0; i < ids.length; i++) { + assertEquals(pieces.get(i).id(), ids[i]); + } + } +} From f55fbec081ae7fdc9b6a90f990539dc9ba1803e7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 06:50:15 -0400 Subject: [PATCH 14/82] OPENNLP-1885: Align null contracts and annotations with the review conventions --- .../tools/tokenize/WordpieceEncoder.java | 21 ++++++++++++++++--- .../sentencepiece/SentencePieceTokenizer.java | 2 ++ .../subword/sentencepiece/PieceTrieTest.java | 3 ++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java index b130f45dca..0fa17ebed8 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; +import opennlp.tools.commons.ThreadSafe; + /** * A {@link SubwordTokenizer} running the full BERT tokenization pipeline of the reference * implementation: basic tokenization (control removal, whitespace normalization, CJK @@ -47,6 +49,7 @@ * * @see WordpieceTokenizer */ +@ThreadSafe public final class WordpieceEncoder implements SubwordTokenizer { // The wordpiece vocabulary convention: a piece with this prefix continues the current word, @@ -71,6 +74,8 @@ public final class WordpieceEncoder implements SubwordTokenizer { * * @param vocabulary The ordered vocabulary; a piece's id is its index. Must not be null, * must not contain nulls or duplicates. + * @throws IllegalArgumentException Thrown if the vocabulary is null, contains a null or + * duplicate entry, or a BERT special token is missing from it. */ public WordpieceEncoder(List vocabulary) { this(vocabulary, true); @@ -83,6 +88,8 @@ public WordpieceEncoder(List vocabulary) { * must not contain nulls or duplicates. * @param lowerCase True for uncased models (lower casing and accent stripping), false for * cased models. + * @throws IllegalArgumentException Thrown if the vocabulary is null, contains a null or + * duplicate entry, or a BERT special token is missing from it. */ public WordpieceEncoder(List vocabulary, boolean lowerCase) { this(vocabulary, lowerCase, WordpieceTokenizer.BERT_CLS_TOKEN, @@ -126,9 +133,17 @@ public WordpieceEncoder(List vocabulary, boolean lowerCase, public WordpieceEncoder(Map vocabularyIds, boolean lowerCase, String classificationToken, String separatorToken, String unknownToken) { - if (vocabularyIds == null || classificationToken == null || separatorToken == null - || unknownToken == null) { - throw new IllegalArgumentException("The vocabulary and special tokens must not be null."); + if (vocabularyIds == null) { + throw new IllegalArgumentException("vocabularyIds must not be null."); + } + if (classificationToken == null) { + throw new IllegalArgumentException("classificationToken must not be null."); + } + if (separatorToken == null) { + throw new IllegalArgumentException("separatorToken must not be null."); + } + if (unknownToken == null) { + throw new IllegalArgumentException("unknownToken must not be null."); } final Map byPiece = new HashMap<>(vocabularyIds.size() * 2); for (final Map.Entry entry : vocabularyIds.entrySet()) { diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 4d73f54016..cdf2231fe4 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.function.IntUnaryOperator; +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.tokenize.SubwordPiece; import opennlp.tools.tokenize.SubwordTokenizer; import opennlp.tools.util.normalizer.AlignedText; @@ -62,6 +63,7 @@ * "SentencePiece: A simple and language independent subword tokenizer and detokenizer for * Neural Text Processing" */ +@ThreadSafe public final class SentencePieceTokenizer implements SubwordTokenizer, OffsetAwareNormalizer { // Serializable through the OffsetAwareNormalizer contract. diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java index bbde60825c..adaf2a292e 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/PieceTrieTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; /** @@ -77,7 +78,7 @@ void testStepsMatchAMapBackedReferenceOverRandomVocabularies() { final boolean anyKeyHasPrefix = keys.stream().anyMatch(k -> k.startsWith(prefix)); if (node == PieceTrie.DEAD) { - assertEquals(false, anyKeyHasPrefix, "dead end despite live prefix: " + prefix); + assertFalse(anyKeyHasPrefix, "dead end despite live prefix: " + prefix); break; } final Integer expected = reference.get(prefix); From 8ebca7eaff3ba60ae3725d55a41b3467b1d25f8e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 07:55:02 -0400 Subject: [PATCH 15/82] OPENNLP-1885: Address review: checked InvalidFormatException for malformed models, tag helpers, javadoc throws --- .../tools/tokenize/WordpieceTokenizer.java | 7 +- .../subword/sentencepiece/ByteBuilder.java | 14 ++- .../sentencepiece/DoubleArrayTrie.java | 3 + .../subword/sentencepiece/IntBuilder.java | 12 +- .../sentencepiece/ModelProtoReader.java | 103 +++++++++++------- .../subword/sentencepiece/PieceTrie.java | 8 +- .../SentencePieceNormalizer.java | 4 +- .../sentencepiece/SentencePieceTokenizer.java | 29 ++--- .../SentencePieceModelValidationTest.java | 11 +- 9 files changed, 123 insertions(+), 68 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java index 97e240e87c..ff7f3a245f 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java @@ -34,8 +34,9 @@ * of BERT tokenization. It does not normalize the input text: no lower casing, * no accent stripping, no control character removal. Text that does not match * the vocabulary's casing - for uncased models that includes every capitalized - * word - is mapped to the unknown token. Use {@link BertTokenizer} for the - * full BERT tokenization pipeline. + * word - is mapped to the unknown token. Use {@link WordpieceEncoder} for the + * full BERT tokenization pipeline; it subsumes the {@code BertTokenizer} + * class shipped in the 3.0.0 milestone builds. *

* As of OpenNLP 3.0.0 the behavior matches the reference BERT wordpiece * implementation in three respects that differ from earlier releases: @@ -58,7 +59,7 @@ * * * - * @see BertTokenizer + * @see WordpieceEncoder */ public class WordpieceTokenizer implements Tokenizer { diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java index ef08bb624a..432f2104dc 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ByteBuilder.java @@ -21,6 +21,9 @@ /** A growable byte buffer supporting append, truncate, and suffix comparison. */ final class ByteBuilder { + /** The smallest backing array, so tiny requested capacities still grow geometrically. */ + private static final int MIN_CAPACITY = 16; + private byte[] data; private int length; @@ -30,7 +33,7 @@ final class ByteBuilder { * @param capacity The initial capacity hint. */ ByteBuilder(int capacity) { - data = new byte[Math.max(capacity, 16)]; + data = new byte[Math.max(capacity, MIN_CAPACITY)]; } /** @@ -40,7 +43,7 @@ final class ByteBuilder { */ void append(byte b) { if (length == data.length) { - data = Arrays.copyOf(data, data.length + (data.length >> 1)); + data = Arrays.copyOf(data, grownLength()); } data[length++] = b; } @@ -54,12 +57,17 @@ void append(byte b) { */ void append(byte[] source, int from, int count) { while (length + count > data.length) { - data = Arrays.copyOf(data, data.length + (data.length >> 1)); + data = Arrays.copyOf(data, grownLength()); } System.arraycopy(source, from, data, length, count); length += count; } + /** {@return the next backing-array length under the 1.5x growth policy} */ + private int grownLength() { + return data.length + (data.length >> 1); + } + /** {@return the number of valid bytes} */ int length() { return length; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java index 28eefd3f1f..75f48e1de8 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java @@ -53,6 +53,7 @@ final class DoubleArrayTrie implements Serializable { * @param data The bytes holding the units; must not be null. * @param offset The offset of the first unit byte. * @param length The number of bytes; must be a positive multiple of four. + * @throws IllegalArgumentException Thrown if {@code length} is not a positive multiple of four. */ DoubleArrayTrie(byte[] data, int offset, int length) { if (length <= 0 || (length & 3) != 0) { @@ -75,6 +76,8 @@ final class DoubleArrayTrie implements Serializable { * @param to The exclusive end of the query window. * @return {@code (value << 32) | matchedLength} for the longest match, or {@code -1} when no * key matches. Values are non-negative, so the result is negative only on no-match. + * @throws IllegalArgumentException Thrown if the trie data references a unit outside its + * bounds, indicating corrupt data. */ long longestPrefixMatch(byte[] key, int from, int to) { // The JVM's own bounds checks guard the walk; the catch below translates an out-of-range unit diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java index 2da34c29b6..b06fb73209 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/IntBuilder.java @@ -21,6 +21,9 @@ /** A growable int buffer supporting append, indexed read, and truncate. */ final class IntBuilder { + /** The smallest backing array, so tiny requested capacities still grow geometrically. */ + private static final int MIN_CAPACITY = 16; + private int[] data; private int length; @@ -30,7 +33,7 @@ final class IntBuilder { * @param capacity The initial capacity hint. */ IntBuilder(int capacity) { - data = new int[Math.max(capacity, 16)]; + data = new int[Math.max(capacity, MIN_CAPACITY)]; } /** @@ -40,11 +43,16 @@ final class IntBuilder { */ void append(int value) { if (length == data.length) { - data = Arrays.copyOf(data, data.length + (data.length >> 1)); + data = Arrays.copyOf(data, grownLength()); } data[length++] = value; } + /** {@return the next backing-array length under the 1.5x growth policy} */ + private int grownLength() { + return data.length + (data.length >> 1); + } + /** * Reads a value by index. * diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java index c4c2a1472e..aff6a194cd 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; +import opennlp.tools.util.InvalidFormatException; + /** * Reads the binary {@code ModelProto} serialization of a SentencePiece {@code .model} file. * @@ -87,9 +89,10 @@ private ModelProtoReader(byte[] data) { * * @param data The raw bytes of a {@code .model} file; must not be null. * @return The parsed model description. - * @throws IllegalArgumentException Thrown if the bytes are not a well-formed model. + * @throws IllegalArgumentException Thrown if {@code data} is null. + * @throws InvalidFormatException Thrown if the bytes are not a well-formed model. */ - static RawModel read(byte[] data) { + static RawModel read(byte[] data) throws InvalidFormatException { if (data == null) { throw new IllegalArgumentException("The model data must not be null."); } @@ -97,8 +100,7 @@ static RawModel read(byte[] data) { final RawModel model = new RawModel(); while (reader.pos < data.length) { final long tag = reader.varint(); - final int field = (int) (tag >>> 3); - switch (field) { + switch (fieldOf(tag)) { case FIELD_MODEL_PIECES -> reader.piece(model, reader.lenPayload(tag)); case FIELD_MODEL_TRAINER_SPEC -> reader.trainerSpec(model, reader.lenPayload(tag)); case FIELD_MODEL_NORMALIZER_SPEC -> reader.normalizerSpec(model, reader.lenPayload(tag)); @@ -107,24 +109,46 @@ static RawModel read(byte[] data) { } } if (model.pieces.isEmpty()) { - throw new IllegalArgumentException("The model defines no pieces."); + throw new InvalidFormatException("The model defines no pieces."); } return model; } + /** + * Extracts the field number from a wire-format tag. + * + * @param tag The field tag. + * @return The field number. + */ + private static int fieldOf(long tag) { + return (int) (tag >>> 3); + } + + /** + * Extracts the wire type from a wire-format tag. + * + * @param tag The field tag. + * @return The wire type. + */ + private static int wireTypeOf(long tag) { + return (int) (tag & 7); + } + /** * Parses one {@code SentencePiece} sub-message and appends its piece, score, and type. * * @param model The model to append to. * @param end The exclusive end offset of the sub-message payload. + * @throws InvalidFormatException Thrown if the sub-message is malformed or defines an empty + * piece or a non-finite score. */ - private void piece(RawModel model, int end) { + private void piece(RawModel model, int end) throws InvalidFormatException { String piece = null; float score = 0; int type = RawModel.TYPE_NORMAL; while (pos < end) { final long tag = varint(); - switch ((int) (tag >>> 3)) { + switch (fieldOf(tag)) { case FIELD_PIECE_PIECE -> piece = utf8(lenPayload(tag)); case FIELD_PIECE_SCORE -> score = fixed32Float(tag); case FIELD_PIECE_TYPE -> type = (int) varintOf(tag); @@ -132,11 +156,11 @@ private void piece(RawModel model, int end) { } } if (piece == null || piece.isEmpty()) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "The model contains an empty piece at index " + model.pieces.size() + "."); } if (Float.isNaN(score) || Float.isInfinite(score)) { - throw new IllegalArgumentException("The score of piece '" + piece + "' is not finite."); + throw new InvalidFormatException("The score of piece '" + piece + "' is not finite."); } model.pieces.add(piece); model.scores.add(score); @@ -148,11 +172,12 @@ private void piece(RawModel model, int end) { * * @param model The model to populate. * @param end The exclusive end offset of the sub-message payload. + * @throws InvalidFormatException Thrown if the sub-message is malformed. */ - private void trainerSpec(RawModel model, int end) { + private void trainerSpec(RawModel model, int end) throws InvalidFormatException { while (pos < end) { final long tag = varint(); - switch ((int) (tag >>> 3)) { + switch (fieldOf(tag)) { case FIELD_TRAINER_MODEL_TYPE -> model.modelType = (int) varintOf(tag); case FIELD_TRAINER_TREAT_WHITESPACE_AS_SUFFIX -> model.treatWhitespaceAsSuffix = varintOf(tag) != 0; @@ -169,11 +194,12 @@ private void trainerSpec(RawModel model, int end) { * * @param model The model to populate. * @param end The exclusive end offset of the sub-message payload. + * @throws InvalidFormatException Thrown if the sub-message is malformed. */ - private void normalizerSpec(RawModel model, int end) { + private void normalizerSpec(RawModel model, int end) throws InvalidFormatException { while (pos < end) { final long tag = varint(); - switch ((int) (tag >>> 3)) { + switch (fieldOf(tag)) { case FIELD_NORMALIZER_PRECOMPILED_CHARSMAP -> model.precompiledCharsMap = bytes(lenPayload(tag)); case FIELD_NORMALIZER_ADD_DUMMY_PREFIX -> model.addDummyPrefix = varintOf(tag) != 0; @@ -191,17 +217,18 @@ private void normalizerSpec(RawModel model, int end) { * * @param model The model to populate. * @param end The exclusive end offset of the sub-message payload. + * @throws InvalidFormatException Thrown if the sub-message is malformed. */ - private void selfTestData(RawModel model, int end) { + private void selfTestData(RawModel model, int end) throws InvalidFormatException { while (pos < end) { final long tag = varint(); - if ((int) (tag >>> 3) == FIELD_SELF_TEST_SAMPLES) { + if (fieldOf(tag) == FIELD_SELF_TEST_SAMPLES) { final int sampleEnd = lenPayload(tag); String input = null; String expected = null; while (pos < sampleEnd) { final long sampleTag = varint(); - switch ((int) (sampleTag >>> 3)) { + switch (fieldOf(sampleTag)) { case FIELD_SAMPLE_INPUT -> input = utf8(lenPayload(sampleTag)); case FIELD_SAMPLE_EXPECTED -> expected = utf8(lenPayload(sampleTag)); default -> skip(sampleTag); @@ -223,12 +250,12 @@ private void selfTestData(RawModel model, int end) { * * @param tag The field tag, whose wire type must be length-delimited. * @return The exclusive end offset of the payload. - * @throws IllegalArgumentException Thrown if the wire type is wrong or the length runs past the + * @throws InvalidFormatException Thrown if the wire type is wrong or the length runs past the * input. */ - private int lenPayload(long tag) { - if ((tag & 7) != WIRE_LEN) { - throw malformed("field " + (tag >>> 3) + " is not length-delimited"); + private int lenPayload(long tag) throws InvalidFormatException { + if (wireTypeOf(tag) != WIRE_LEN) { + throw malformed("field " + fieldOf(tag) + " is not length-delimited"); } final long length = varint(); if (length < 0 || pos + length > data.length) { @@ -242,11 +269,11 @@ private int lenPayload(long tag) { * * @param tag The field tag, whose wire type must be varint. * @return The decoded value. - * @throws IllegalArgumentException Thrown if the wire type is wrong or the varint is malformed. + * @throws InvalidFormatException Thrown if the wire type is wrong or the varint is malformed. */ - private long varintOf(long tag) { - if ((tag & 7) != WIRE_VARINT) { - throw malformed("field " + (tag >>> 3) + " is not a varint"); + private long varintOf(long tag) throws InvalidFormatException { + if (wireTypeOf(tag) != WIRE_VARINT) { + throw malformed("field " + fieldOf(tag) + " is not a varint"); } return varint(); } @@ -256,11 +283,11 @@ private long varintOf(long tag) { * * @param tag The field tag, whose wire type must be 32-bit. * @return The decoded float. - * @throws IllegalArgumentException Thrown if the wire type is wrong or the input is truncated. + * @throws InvalidFormatException Thrown if the wire type is wrong or the input is truncated. */ - private float fixed32Float(long tag) { - if ((tag & 7) != WIRE_FIXED32) { - throw malformed("field " + (tag >>> 3) + " is not a 32-bit value"); + private float fixed32Float(long tag) throws InvalidFormatException { + if (wireTypeOf(tag) != WIRE_FIXED32) { + throw malformed("field " + fieldOf(tag) + " is not a 32-bit value"); } if (pos + 4 > data.length) { throw malformed("truncated 32-bit value"); @@ -300,10 +327,10 @@ private byte[] bytes(int end) { * Reads a base-128 varint from the current position, advancing past it. * * @return The decoded value. - * @throws IllegalArgumentException Thrown if the input ends mid-varint or the varint exceeds 64 + * @throws InvalidFormatException Thrown if the input ends mid-varint or the varint exceeds 64 * bits. */ - private long varint() { + private long varint() throws InvalidFormatException { long value = 0; for (int shift = 0; shift < 64; shift += 7) { if (pos >= data.length) { @@ -322,16 +349,16 @@ private long varint() { * Skips the value of an unrecognized field according to its wire type. * * @param tag The field tag. - * @throws IllegalArgumentException Thrown if the wire type is unsupported or the value runs past + * @throws InvalidFormatException Thrown if the wire type is unsupported or the value runs past * the input. */ - private void skip(long tag) { - switch ((int) (tag & 7)) { + private void skip(long tag) throws InvalidFormatException { + switch (wireTypeOf(tag)) { case WIRE_VARINT -> varint(); case WIRE_FIXED64 -> advance(8); case WIRE_LEN -> pos = lenPayload(tag); case WIRE_FIXED32 -> advance(4); - default -> throw malformed("unsupported wire type " + (tag & 7)); + default -> throw malformed("unsupported wire type " + wireTypeOf(tag)); } } @@ -339,9 +366,9 @@ private void skip(long tag) { * Advances the position by a fixed number of bytes. * * @param count The number of bytes to skip. - * @throws IllegalArgumentException Thrown if fewer than {@code count} bytes remain. + * @throws InvalidFormatException Thrown if fewer than {@code count} bytes remain. */ - private void advance(int count) { + private void advance(int count) throws InvalidFormatException { if (pos + count > data.length) { throw malformed("truncated field"); } @@ -354,8 +381,8 @@ private void advance(int count) { * @param detail A short description of what is malformed. * @return The exception to throw. */ - private IllegalArgumentException malformed(String detail) { - return new IllegalArgumentException( + private InvalidFormatException malformed(String detail) { + return new InvalidFormatException( "The model data is malformed at byte " + pos + ": " + detail + "."); } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java index 5253a94675..42477251ab 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/PieceTrie.java @@ -40,6 +40,9 @@ final class PieceTrie implements Serializable { // this; otherwise a linear scan of the sorted label slice is used. private static final int DIRECT_THRESHOLD = 8; + /** The width of a wide node's direct dispatch slice. */ + private static final int DIRECT_TABLE_SIZE = 256; + // Per node: the slice [childStart[n], childStart[n + 1]) of labels/childNodes, and the piece id // accepted at the node, or -1. Wide nodes additionally index directPool at directStart[n]. private final int[] childStart; @@ -67,13 +70,13 @@ private PieceTrie(int[] childStart, byte[] labels, int[] childNodes, int[] value int wide = 0; for (int node = 0; node < values.length; node++) { if (childStart[node + 1] - childStart[node] > DIRECT_THRESHOLD) { - directStart[node] = wide * 256; + directStart[node] = wide * DIRECT_TABLE_SIZE; wide++; } else { directStart[node] = -1; } } - this.directPool = new int[wide * 256]; + this.directPool = new int[wide * DIRECT_TABLE_SIZE]; Arrays.fill(directPool, DEAD); for (int node = 0; node < values.length; node++) { final int direct = directStart[node]; @@ -91,6 +94,7 @@ private PieceTrie(int[] childStart, byte[] labels, int[] childNodes, int[] value * @param pieces The UTF-8 bytes of each piece; must not be null or contain empty keys. * @param ids The id stored for each piece, parallel to {@code pieces}. * @return The packed trie. + * @throws IllegalArgumentException Thrown if a piece is defined more than once. */ static PieceTrie build(byte[][] pieces, int[] ids) { final Integer[] order = new Integer[pieces.length]; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java index 0dd83c2784..5494282e95 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -151,7 +151,7 @@ Normalized normalize(byte[] input, int inputLength) { int from = 0; int consumed = 0; - // Ignores heading whitespace. + // Ignores leading whitespace. if (removeExtraWhitespaces) { while (from < inputLength) { normalizePrefix(input, inputLength, from, chunk); @@ -194,7 +194,7 @@ Normalized normalize(byte[] input, int inputLength) { final int spTo = chunk.to; final byte[] spData = chunk.data; - // Removes heading spaces in the chunk if the previous chunk ended with whitespace. + // Removes leading spaces in the chunk if the previous chunk ended with whitespace. while (isPrevSpace && spFrom < spTo && spData[spFrom] == ' ') { spFrom++; } diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index cdf2231fe4..07a52e8fae 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -36,6 +36,7 @@ import opennlp.tools.commons.ThreadSafe; import opennlp.tools.tokenize.SubwordPiece; import opennlp.tools.tokenize.SubwordTokenizer; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.util.normalizer.AlignedText; import opennlp.tools.util.normalizer.Alignment; import opennlp.tools.util.normalizer.OffsetAwareNormalizer; @@ -107,9 +108,9 @@ public enum Algorithm { * table, the normalizer, and the encoder matching the model's algorithm. * * @param model The parsed model description. - * @throws IllegalArgumentException Thrown if the model is structurally invalid. + * @throws InvalidFormatException Thrown if the model is structurally invalid. */ - private SentencePieceTokenizer(ModelProtoReader.RawModel model) { + private SentencePieceTokenizer(ModelProtoReader.RawModel model) throws InvalidFormatException { final int count = model.pieces.size(); pieces = model.pieces.toArray(new String[0]); scores = new float[count]; @@ -122,7 +123,7 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { algorithm = switch (model.modelType) { case ModelProtoReader.RawModel.MODEL_TYPE_UNIGRAM -> Algorithm.UNIGRAM; case ModelProtoReader.RawModel.MODEL_TYPE_BPE -> Algorithm.BPE; - default -> throw new IllegalArgumentException( + default -> throw new InvalidFormatException( "The model type " + model.modelType + " is not supported; only the unigram and BPE" + " algorithms are."); }; @@ -139,11 +140,11 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { for (int i = 0; i < count; i++) { final String piece = pieces[i]; if (piece.length() >= MAX_PIECE_LENGTH) { - throw new IllegalArgumentException("The piece with id " + i + " is longer than " + throw new InvalidFormatException("The piece with id " + i + " is longer than " + MAX_PIECE_LENGTH + " characters."); } if (piece.indexOf(0) >= 0) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "The piece with id " + i + " contains a null character."); } final boolean isMain = @@ -151,7 +152,7 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { final Map target = isMain || algorithm == Algorithm.BPE ? mainPieces : reservedPieces; if (mainPieces.containsKey(piece) || reservedPieces.containsKey(piece)) { - throw PieceTrie.duplicatePiece(piece); + throw new InvalidFormatException(PieceTrie.duplicatePiece(piece).getMessage()); } target.put(piece, i); switch (types[i]) { @@ -159,18 +160,18 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { case TYPE_USER_DEFINED -> userDefined.add(piece); case TYPE_UNKNOWN -> { if (foundUnkId >= 0) { - throw new IllegalArgumentException("The model defines more than one unknown piece."); + throw new InvalidFormatException("The model defines more than one unknown piece."); } foundUnkId = i; } case TYPE_BYTE -> { if (!byteFallback) { - throw new IllegalArgumentException("The model defines the byte piece '" + piece + throw new InvalidFormatException("The model defines the byte piece '" + piece + "' although byte fallback is disabled."); } final int b = parseBytePiece(piece); if (b < 0) { - throw new IllegalArgumentException("The byte piece '" + piece + "' is invalid."); + throw new InvalidFormatException("The byte piece '" + piece + "' is invalid."); } byteToId[b] = i; } @@ -180,13 +181,13 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) { } } if (foundUnkId < 0) { - throw new IllegalArgumentException("The model defines no unknown piece."); + throw new InvalidFormatException("The model defines no unknown piece."); } unkId = foundUnkId; if (byteFallback) { for (int b = 0; b < 256; b++) { if (byteToId[b] < 0) { - throw new IllegalArgumentException("The model enables byte fallback but defines no" + throw new InvalidFormatException("The model enables byte fallback but defines no" + " piece for byte " + b + "."); } } @@ -255,7 +256,8 @@ private static PieceTrie trieOf(List pieceList, IntUnaryOperator idOf) { * @param modelFile The {@code .model} file to load; must not be null. * @return The ready-to-use tokenizer. * @throws IOException Thrown if the file cannot be read. - * @throws IllegalArgumentException Thrown if the file is not a valid model. + * @throws InvalidFormatException Thrown if the content is not a valid model. + * @throws IllegalArgumentException Thrown if {@code modelFile} is null. */ public static SentencePieceTokenizer load(Path modelFile) throws IOException { if (modelFile == null) { @@ -271,7 +273,8 @@ public static SentencePieceTokenizer load(Path modelFile) throws IOException { * null. * @return The ready-to-use tokenizer. * @throws IOException Thrown if the stream cannot be read. - * @throws IllegalArgumentException Thrown if the bytes are not a valid model. + * @throws InvalidFormatException Thrown if the content is not a valid model. + * @throws IllegalArgumentException Thrown if {@code in} is null. */ public static SentencePieceTokenizer load(InputStream in) throws IOException { if (in == null) { diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java index a65df08991..48c4179cdd 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java @@ -33,6 +33,7 @@ import org.junit.jupiter.api.Test; import opennlp.tools.tokenize.SubwordPiece; +import opennlp.tools.util.InvalidFormatException; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -50,14 +51,14 @@ void testNullAndEmptyInputFailLoudly() { () -> SentencePieceTokenizer.load((Path) null)); assertThrows(IllegalArgumentException.class, () -> SentencePieceTokenizer.load((InputStream) null)); - assertThrows(IllegalArgumentException.class, + assertThrows(InvalidFormatException.class, () -> SentencePieceTokenizer.load(new ByteArrayInputStream(new byte[0]))); } @Test void testGarbageBytesFailLoudly() { final byte[] garbage = "this is not a model file at all".getBytes(StandardCharsets.UTF_8); - assertThrows(IllegalArgumentException.class, + assertThrows(InvalidFormatException.class, () -> SentencePieceTokenizer.load(new ByteArrayInputStream(garbage))); } @@ -65,7 +66,7 @@ void testGarbageBytesFailLoudly() { void testTruncatedModelFailsLoudly() throws IOException { final byte[] whole = readModel(); final byte[] truncated = Arrays.copyOf(whole, whole.length / 3); - assertThrows(IllegalArgumentException.class, + assertThrows(InvalidFormatException.class, () -> SentencePieceTokenizer.load(new ByteArrayInputStream(truncated))); } @@ -73,7 +74,7 @@ void testTruncatedModelFailsLoudly() throws IOException { void testUnsupportedModelTypeFailsLoudly() { // A minimal well-formed model claiming the WORD algorithm (model_type = 3). final byte[] model = minimalModel(3); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> SentencePieceTokenizer.load(new ByteArrayInputStream(model))); assertTrue(e.getMessage().contains("not supported"), e.getMessage()); } @@ -81,7 +82,7 @@ void testUnsupportedModelTypeFailsLoudly() { @Test void testMissingUnknownPieceFailsLoudly() { final byte[] model = minimalModelWithoutUnk(); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> SentencePieceTokenizer.load(new ByteArrayInputStream(model))); assertTrue(e.getMessage().contains("unknown piece"), e.getMessage()); } From 80029caecc8f4a23efb49be425d4f877763e9b95 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:00:46 -0400 Subject: [PATCH 16/82] OPENNLP-1885: Address review: validation message style, shared test fixtures, thread safety wording - Normalize the argument validation messages to the project style, naming the offending parameter and dropping the leading article and the trailing period, in WordpieceEncoder, SentencePieceTokenizer, ModelProtoReader, BpeEncoder, UnigramEncoder and the SubwordPiece compact constructor. - Stop promising thread safety in the SubwordTokenizer contract and state that it is implementation specific instead; the manual now records that both shipped implementations are immutable and therefore safe for concurrent use. - Drop the @throws IllegalArgumentException tags that only repeated the inherited contract on the normalize and normalizeAligned overrides, leaving a plain {@inheritDoc} as the rest of the class does. - Remove commentary about release history rather than about the code: the pointer to the BertTokenizer class of the 3.0.0 milestone builds in WordpieceTokenizer, and the "frozen" qualifier on the ReferenceBertPipeline baseline. - Move the bundled model loading and the fixture file reading out of SentencePieceParityTest into SentencePieceFixtures, so the alignment, validation and serialization tests no longer reach into another test class for a tokenizer. - Extract MODEL_SUFFIX and FIXTURES_SUFFIX constants on SentencePieceFixtures and use them in SentencePieceRealModelEvalTest when deriving a fixture path from a model path, instead of repeating the two literals. - Fold the five duplicated @ValueSource model lists into a single SentencePieceFixtures#models @MethodSource, so adding a bundled model stays a one line change. - Correct the parity test javadoc, which credited a nonexistent gen_fixtures.tsv sibling script instead of the gen_fixtures.py script in the test resources. - Pin accessors that had no coverage: every score is finite and out of range ids are rejected, byte pieces occur only in byte fallback models and always render in the <0x..> form, and isByte rejects negative ids. - Assert SubwordPiece.span() next to start and end in WordpieceEncoderTest so the derived span stays covered by the piece assertions. - Document the IOException of the serialized helper in the serialization test and fully qualify the OutputStream javadoc link now that the import is gone. --- .../opennlp/tools/tokenize/SubwordPiece.java | 2 +- .../tools/tokenize/SubwordTokenizer.java | 3 +- .../tools/tokenize/WordpieceEncoder.java | 12 ++-- .../tools/tokenize/WordpieceTokenizer.java | 3 +- .../tools/tokenize/ReferenceBertPipeline.java | 6 +- .../tools/tokenize/WordpieceEncoderTest.java | 3 + opennlp-docs/src/docbkx/tokenizer.xml | 5 +- .../subword/sentencepiece/BpeEncoder.java | 2 +- .../sentencepiece/ModelProtoReader.java | 2 +- .../sentencepiece/SentencePieceTokenizer.java | 28 +++----- .../subword/sentencepiece/UnigramEncoder.java | 2 +- .../SentencePieceAlignmentTest.java | 2 +- .../sentencepiece/SentencePieceFixtures.java | 67 +++++++++++++++++-- .../SentencePieceModelValidationTest.java | 37 +++++++++- .../SentencePieceParityTest.java | 47 +++---------- .../SentencePieceRealModelEvalTest.java | 11 +-- ...ntencePieceTokenizerSerializationTest.java | 19 +++--- 17 files changed, 154 insertions(+), 97 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java index 032d04f1c3..62260b7929 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java @@ -43,7 +43,7 @@ public record SubwordPiece(String piece, int id, int start, int end) { */ public SubwordPiece { if (piece == null || piece.isEmpty()) { - throw new IllegalArgumentException("The piece must not be null or empty."); + throw new IllegalArgumentException("piece must not be null or empty"); } if (start < 0 || end < start) { throw new IllegalArgumentException( diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java index bd3cdf3984..957ff62229 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java @@ -26,8 +26,7 @@ * model's normalized form, so a piece is generally not a substring of the input. The offsets * carried by each {@link SubwordPiece} always refer to the caller's original text.

* - *

Implementations are expected to be safe for concurrent use by multiple threads; any - * implementation that is not must document it.

+ *

Thread safety is implementation specific.

*/ public interface SubwordTokenizer { diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java index 0fa17ebed8..49a7910e9c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java @@ -134,16 +134,16 @@ public WordpieceEncoder(Map vocabularyIds, boolean lowerCase, String classificationToken, String separatorToken, String unknownToken) { if (vocabularyIds == null) { - throw new IllegalArgumentException("vocabularyIds must not be null."); + throw new IllegalArgumentException("vocabularyIds must not be null"); } if (classificationToken == null) { - throw new IllegalArgumentException("classificationToken must not be null."); + throw new IllegalArgumentException("classificationToken must not be null"); } if (separatorToken == null) { - throw new IllegalArgumentException("separatorToken must not be null."); + throw new IllegalArgumentException("separatorToken must not be null"); } if (unknownToken == null) { - throw new IllegalArgumentException("unknownToken must not be null."); + throw new IllegalArgumentException("unknownToken must not be null"); } final Map byPiece = new HashMap<>(vocabularyIds.size() * 2); for (final Map.Entry entry : vocabularyIds.entrySet()) { @@ -175,7 +175,7 @@ public WordpieceEncoder(Map vocabularyIds, boolean lowerCase, */ private static Map byPiece(List vocabulary) { if (vocabulary == null) { - throw new IllegalArgumentException("The vocabulary must not be null."); + throw new IllegalArgumentException("vocabulary must not be null"); } final Map byPiece = new HashMap<>(vocabulary.size() * 2); for (int id = 0; id < vocabulary.size(); id++) { @@ -212,7 +212,7 @@ private static int requiredId(Map ids, String specialToken) { @Override public List encode(CharSequence text) { if (text == null) { - throw new IllegalArgumentException("The text must not be null."); + throw new IllegalArgumentException("text must not be null"); } final String original = text.toString(); diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java index ff7f3a245f..fa1014ab2c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceTokenizer.java @@ -35,8 +35,7 @@ * no accent stripping, no control character removal. Text that does not match * the vocabulary's casing - for uncased models that includes every capitalized * word - is mapped to the unknown token. Use {@link WordpieceEncoder} for the - * full BERT tokenization pipeline; it subsumes the {@code BertTokenizer} - * class shipped in the 3.0.0 milestone builds. + * full BERT tokenization pipeline. *

* As of OpenNLP 3.0.0 the behavior matches the reference BERT wordpiece * implementation in three respects that differ from earlier releases: diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java index a07d6faafc..ca7b2b5618 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java @@ -21,9 +21,9 @@ import java.util.Set; /** - * The reference BERT basic-tokenization stage feeding {@link WordpieceTokenizer}, kept - * test-only as the frozen differential baseline for {@link WordpieceEncoderTest}: the - * encoder's piece sequence must match this pipeline exactly. + * The reference BERT basic-tokenization stage feeding {@link WordpieceTokenizer}, serving as + * the differential baseline for {@link WordpieceEncoderTest}: the encoder's piece sequence + * must match this pipeline exactly. */ final class ReferenceBertPipeline { diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java index cf545b183f..0cc97d3b6c 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java @@ -27,6 +27,8 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.util.Span; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -57,6 +59,7 @@ private static void assertPiece(SubwordPiece piece, String expectedPiece, int ex assertEquals(expectedId, piece.id()); assertEquals(expectedStart, piece.start(), "start of " + piece); assertEquals(expectedEnd, piece.end(), "end of " + piece); + assertEquals(new Span(expectedStart, expectedEnd), piece.span(), "span of " + piece); } /** diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index fa7a509f8e..0c84b5c855 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -551,8 +551,9 @@ wordTokenizer.tokenize("The quick brown fox.", (start, end, type) -> { generally not a substring of the input; the spans always refer to the caller's original text, so annotations computed over the pieces can be mapped back without guesswork. The encodeToIds and encodeToPieces methods return just the ids or - the piece strings when the spans are not needed. Implementations are expected to be safe - for concurrent use by multiple threads. + the piece strings when the spans are not needed. Thread safety is implementation specific; + both implementations described below are immutable and safe for concurrent use by + multiple threads.

SentencePiece diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java index 0a08c980eb..6e448e5c38 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java @@ -84,7 +84,7 @@ private record Pair(int left, int right, float score, int size) { */ List encode(byte[] normalized, int size) { if (normalized == null) { - throw new IllegalArgumentException("The normalized buffer must not be null."); + throw new IllegalArgumentException("normalized must not be null"); } if (size == 0) { return List.of(); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java index aff6a194cd..a6ceca6b84 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java @@ -94,7 +94,7 @@ private ModelProtoReader(byte[] data) { */ static RawModel read(byte[] data) throws InvalidFormatException { if (data == null) { - throw new IllegalArgumentException("The model data must not be null."); + throw new IllegalArgumentException("data must not be null"); } final ModelProtoReader reader = new ModelProtoReader(data); final RawModel model = new RawModel(); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 07a52e8fae..041cd7b285 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -261,7 +261,7 @@ private static PieceTrie trieOf(List pieceList, IntUnaryOperator idOf) { */ public static SentencePieceTokenizer load(Path modelFile) throws IOException { if (modelFile == null) { - throw new IllegalArgumentException("The model file must not be null."); + throw new IllegalArgumentException("modelFile must not be null"); } return new SentencePieceTokenizer(ModelProtoReader.read(Files.readAllBytes(modelFile))); } @@ -278,7 +278,7 @@ public static SentencePieceTokenizer load(Path modelFile) throws IOException { */ public static SentencePieceTokenizer load(InputStream in) throws IOException { if (in == null) { - throw new IllegalArgumentException("The input stream must not be null."); + throw new IllegalArgumentException("in must not be null"); } return new SentencePieceTokenizer(ModelProtoReader.read(in.readAllBytes())); } @@ -293,7 +293,7 @@ public static SentencePieceTokenizer load(InputStream in) throws IOException { */ public void serialize(OutputStream out) throws IOException { if (out == null) { - throw new IllegalArgumentException("The output stream must not be null."); + throw new IllegalArgumentException("out must not be null"); } try (ObjectOutputStream oos = new ObjectOutputStream(out)) { oos.writeObject(this); @@ -350,10 +350,10 @@ public static SentencePieceTokenizer deserialize(InputStream in) public static SentencePieceTokenizer deserialize(InputStream in, DeserializationLimits limits) throws IOException, ClassNotFoundException { if (in == null) { - throw new IllegalArgumentException("The input stream must not be null."); + throw new IllegalArgumentException("in must not be null"); } if (limits == null) { - throw new IllegalArgumentException("The limits must not be null."); + throw new IllegalArgumentException("limits must not be null"); } try (ObjectInputStream ois = new ObjectInputStream(in)) { ois.setObjectInputFilter(buildFilter(limits)); @@ -475,7 +475,7 @@ private static ObjectInputFilter buildFilter(DeserializationLimits limits) { @Override public List encode(CharSequence text) { if (text == null) { - throw new IllegalArgumentException("The text must not be null."); + throw new IllegalArgumentException("text must not be null"); } final Utf8Text input = Utf8Text.of(text); final SentencePieceNormalizer.Normalized normalized = @@ -549,25 +549,17 @@ public List encode(CharSequence text) { return out; } - /** - * {@inheritDoc} - * - * @throws IllegalArgumentException Thrown if {@code text} is null. - */ + /** {@inheritDoc} */ @Override public CharSequence normalize(CharSequence text) { return normalizeAligned(text).normalized(); } - /** - * {@inheritDoc} - * - * @throws IllegalArgumentException Thrown if {@code text} is null. - */ + /** {@inheritDoc} */ @Override public AlignedText normalizeAligned(CharSequence text) { if (text == null) { - throw new IllegalArgumentException("The text must not be null."); + throw new IllegalArgumentException("text must not be null"); } final Utf8Text input = Utf8Text.of(text); final SentencePieceNormalizer.Normalized result = @@ -662,7 +654,7 @@ public String idToPiece(int id) { */ public int pieceToId(String piece) { if (piece == null) { - throw new IllegalArgumentException("The piece must not be null."); + throw new IllegalArgumentException("piece must not be null"); } final Integer reserved = reservedPieces.get(piece); if (reserved != null) { diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java index 1efa00310d..5be10c2c84 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/UnigramEncoder.java @@ -75,7 +75,7 @@ final class UnigramEncoder implements Serializable { */ List encode(byte[] normalized, int size) { if (normalized == null) { - throw new IllegalArgumentException("The normalized buffer must not be null."); + throw new IllegalArgumentException("normalized must not be null"); } if (size == 0) { return List.of(); diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java index 41e2a906fc..7ac28e67ba 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java @@ -38,7 +38,7 @@ class SentencePieceAlignmentTest { private static SentencePieceTokenizer unigram() { - return SentencePieceParityTest.tokenizer("tiny-unigram"); + return SentencePieceFixtures.tokenizer("tiny-unigram"); } @ParameterizedTest diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java index 11e54d2bef..a2c06d6177 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceFixtures.java @@ -18,24 +18,83 @@ import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; import opennlp.tools.tokenize.SubwordPiece; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** - * Shared support for the tab-separated parity fixture files produced by the - * {@code gen_fixtures.py} and {@code gen_real_fixtures.py} scripts in the test resources: one - * line per input, holding the input, the expected piece count, four columns per expected piece - * (content, id, start, end), and the expected normalized form. + * Shared support for the bundled test models and for the tab-separated parity fixture files + * produced by the {@code gen_fixtures.py} and {@code gen_real_fixtures.py} scripts in the test + * resources: one line per input, holding the input, the expected piece count, four columns per + * expected piece (content, id, start, end), and the expected normalized form. */ final class SentencePieceFixtures { + /** The file name suffix of a bundled or downloaded SentencePiece model. */ + static final String MODEL_SUFFIX = ".model"; + + /** The file name suffix of the fixture file belonging to a model. */ + static final String FIXTURES_SUFFIX = ".fixtures.tsv"; + + private static final Map LOADED = new ConcurrentHashMap<>(); + private SentencePieceFixtures() { } + /** + * The bundled models, one per algorithm and normalizer variant the reader must handle. + * + * @return The model names, usable as {@code @MethodSource} arguments. + */ + static Stream models() { + return Stream.of("tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", "tiny-unigram-identity", + "tiny-unigram-suffix"); + } + + /** + * Loads a bundled model from the test resources, caching the result so the parsing cost is paid + * once per model across all test classes. + * + * @param model The bundled model name, without the {@link #MODEL_SUFFIX} suffix. + * @return The loaded tokenizer, shared by all callers. + */ + static SentencePieceTokenizer tokenizer(String model) { + return LOADED.computeIfAbsent(model, name -> { + try (InputStream in = + SentencePieceFixtures.class.getResourceAsStream(name + MODEL_SUFFIX)) { + assertNotNull(in, "missing test resource " + name + MODEL_SUFFIX); + return SentencePieceTokenizer.load(in); + } catch (IOException e) { + throw new IllegalStateException(e); + } + }); + } + + /** + * Reads the fixture file belonging to a bundled model. + * + * @param model The bundled model name, without the {@link #MODEL_SUFFIX} suffix. + * @return The parsed fixtures in file order. + * @throws IOException Thrown if the fixture resource cannot be read. + */ + static List fixtures(String model) throws IOException { + try (InputStream in = + SentencePieceFixtures.class.getResourceAsStream(model + FIXTURES_SUFFIX)) { + assertNotNull(in, "missing test resource " + model + FIXTURES_SUFFIX); + return read(new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))); + } + } + /** * One parsed fixture line: an input with the piece sequence and normalized form the reference * implementation produced for it. diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java index 48c4179cdd..a207625290 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java @@ -36,6 +36,7 @@ import opennlp.tools.util.InvalidFormatException; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -89,7 +90,7 @@ void testMissingUnknownPieceFailsLoudly() { @Test void testConcurrentEncodingIsConsistent() throws Exception { - final SentencePieceTokenizer tokenizer = SentencePieceParityTest.tokenizer("tiny-unigram"); + final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram"); final String[] inputs = { "The quick brown fox jumps over the lazy dog.", "tokenization and segmentation", @@ -125,7 +126,7 @@ void testConcurrentEncodingIsConsistent() throws Exception { @Test void testVocabularyAccessors() { - final SentencePieceTokenizer tokenizer = SentencePieceParityTest.tokenizer("tiny-unigram"); + final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram"); assertEquals(300, tokenizer.vocabularySize()); assertEquals(SentencePieceTokenizer.Algorithm.UNIGRAM, tokenizer.algorithm()); for (int id = 0; id < tokenizer.vocabularySize(); id++) { @@ -141,6 +142,38 @@ void testVocabularyAccessors() { assertThrows(IllegalArgumentException.class, () -> tokenizer.pieceToId(null)); } + @Test + void testScoresAreFiniteAndRangeChecked() { + final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram"); + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + assertTrue(Float.isFinite(tokenizer.score(id)), "score of piece " + id); + } + assertThrows(IllegalArgumentException.class, () -> tokenizer.score(-1)); + assertThrows(IllegalArgumentException.class, + () -> tokenizer.score(tokenizer.vocabularySize())); + } + + @Test + void testBytePiecesExistOnlyInByteFallbackModels() { + final SentencePieceTokenizer byteFallback = + SentencePieceFixtures.tokenizer("tiny-unigram-bytefb"); + int bytePieces = 0; + for (int id = 0; id < byteFallback.vocabularySize(); id++) { + if (byteFallback.isByte(id)) { + bytePieces++; + assertTrue(byteFallback.idToPiece(id).startsWith("<0x"), + "byte piece " + id + " is " + byteFallback.idToPiece(id)); + } + } + assertEquals(256, bytePieces, "byte fallback defines one piece per byte value"); + + final SentencePieceTokenizer plain = SentencePieceFixtures.tokenizer("tiny-unigram"); + for (int id = 0; id < plain.vocabularySize(); id++) { + assertFalse(plain.isByte(id), "piece " + id + " must not be a byte piece"); + } + assertThrows(IllegalArgumentException.class, () -> plain.isByte(-1)); + } + private static byte[] readModel() throws IOException { try (InputStream in = SentencePieceModelValidationTest.class.getResourceAsStream("tiny-unigram.model")) { diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java index 95730ef911..f6361a93b2 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java @@ -16,51 +16,30 @@ */ package opennlp.subword.sentencepiece; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.List; -import java.util.Map; import java.util.StringJoiner; -import java.util.concurrent.ConcurrentHashMap; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** * Asserts exact parity with the reference implementation: for every fixture input, the pieces, * ids, original-text spans, and the normalized form must equal what the reference produced for - * the same bundled model. The fixtures were generated by {@code gen_fixtures.tsv}'s sibling - * script (see the test resources) against the sentencepiece Python package. + * the same bundled model. The fixtures were generated by the {@code gen_fixtures.py} script in + * the test resources against the sentencepiece Python package. */ class SentencePieceParityTest { - private static final Map LOADED = new ConcurrentHashMap<>(); - - static SentencePieceTokenizer tokenizer(String model) { - return LOADED.computeIfAbsent(model, name -> { - try (InputStream in = SentencePieceParityTest.class.getResourceAsStream(name + ".model")) { - assertNotNull(in, "missing test resource " + name + ".model"); - return SentencePieceTokenizer.load(in); - } catch (IOException e) { - throw new IllegalStateException(e); - } - }); - } - @ParameterizedTest - @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", - "tiny-unigram-identity", "tiny-unigram-suffix"}) + @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models") void testFixtureParity(String model) throws IOException { - final SentencePieceTokenizer tokenizer = tokenizer(model); + final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer(model); int lines = 0; - for (final SentencePieceFixtures.Fixture fixture : fixtures(model)) { + for (final SentencePieceFixtures.Fixture fixture : SentencePieceFixtures.fixtures(model)) { lines++; SentencePieceFixtures.assertFixture(tokenizer, fixture, model + " input <" + fixture.input() + ">"); @@ -69,10 +48,9 @@ void testFixtureParity(String model) throws IOException { } @ParameterizedTest - @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", - "tiny-unigram-identity", "tiny-unigram-suffix"}) + @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models") void testEmbeddedSelfTestSamples(String model) { - final SentencePieceTokenizer tokenizer = tokenizer(model); + final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer(model); final List inputs = tokenizer.selfTestInputs(); final List expected = tokenizer.selfTestExpected(); assertTrue(!inputs.isEmpty(), "the tiny models embed self-test samples"); @@ -85,13 +63,4 @@ void testEmbeddedSelfTestSamples(String model) { model + " self-test sample <" + inputs.get(i) + ">"); } } - - private static List fixtures(String model) throws IOException { - try (InputStream in = - SentencePieceParityTest.class.getResourceAsStream(model + ".fixtures.tsv")) { - assertNotNull(in, "missing test resource " + model + ".fixtures.tsv"); - return SentencePieceFixtures.read( - new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))); - } - } } diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java index fd1a9ca9ef..1174f8652c 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java @@ -47,10 +47,13 @@ void testRealModelParity() throws IOException { int models = 0; try (Stream files = Files.list(Path.of(dir))) { - for (final Path model : files.filter(f -> f.toString().endsWith(".model")).sorted() - .toList()) { - final Path fixtures = Path.of(model.toString() - .substring(0, model.toString().length() - ".model".length()) + ".fixtures.tsv"); + for (final Path model : files + .filter(f -> f.toString().endsWith(SentencePieceFixtures.MODEL_SUFFIX)) + .sorted().toList()) { + final String path = model.toString(); + final Path fixtures = Path.of( + path.substring(0, path.length() - SentencePieceFixtures.MODEL_SUFFIX.length()) + + SentencePieceFixtures.FIXTURES_SUFFIX); assumeTrue(Files.exists(fixtures), "no fixtures for " + model.getFileName()); models++; assertModel(model, fixtures); diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java index c90e7201ac..e5bb42d148 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceTokenizerSerializationTest.java @@ -27,7 +27,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertIterableEquals; @@ -52,10 +52,9 @@ class SentencePieceTokenizerSerializationTest { }; @ParameterizedTest - @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", - "tiny-unigram-identity", "tiny-unigram-suffix"}) + @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models") void testRoundTripPreservesEncoding(String model) throws IOException, ClassNotFoundException { - final SentencePieceTokenizer original = SentencePieceParityTest.tokenizer(model); + final SentencePieceTokenizer original = SentencePieceFixtures.tokenizer(model); final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { @@ -79,23 +78,23 @@ void testRoundTripPreservesEncoding(String model) throws IOException, ClassNotFo /** * Serializes the tokenizer of the given fixture model through - * {@link SentencePieceTokenizer#serialize(OutputStream)}. + * {@link SentencePieceTokenizer#serialize(java.io.OutputStream)}. * * @param model The fixture model name. * @return The serialized bytes. + * @throws IOException Thrown if the model cannot be read or serialized. */ private static byte[] serialized(String model) throws IOException { final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - SentencePieceParityTest.tokenizer(model).serialize(bytes); + SentencePieceFixtures.tokenizer(model).serialize(bytes); return bytes.toByteArray(); } @ParameterizedTest - @ValueSource(strings = {"tiny-unigram", "tiny-unigram-bytefb", "tiny-bpe", - "tiny-unigram-identity", "tiny-unigram-suffix"}) + @MethodSource("opennlp.subword.sentencepiece.SentencePieceFixtures#models") void testGuardedDeserializePreservesEncoding(String model) throws IOException, ClassNotFoundException { - final SentencePieceTokenizer original = SentencePieceParityTest.tokenizer(model); + final SentencePieceTokenizer original = SentencePieceFixtures.tokenizer(model); final SentencePieceTokenizer copy = SentencePieceTokenizer.deserialize(new ByteArrayInputStream(serialized(model))); @@ -141,7 +140,7 @@ void testStreamExceedingLimitsIsRejected() throws IOException { */ @Test void testNullArgumentsAreRejected() throws IOException { - final SentencePieceTokenizer tokenizer = SentencePieceParityTest.tokenizer("tiny-unigram"); + final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram"); assertThrows(IllegalArgumentException.class, () -> tokenizer.serialize(null)); assertThrows(IllegalArgumentException.class, () -> SentencePieceTokenizer.deserialize(null)); From 397c9ec87b93b90d4db767282202f8383f01eebc Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Wed, 29 Jul 2026 04:38:26 -0400 Subject: [PATCH 17/82] OPENNLP-1885: Deprecate BertTokenizer as a shim over WordpieceEncoder per review Applies the review: the old entry point stays through one stable release instead of being removed, and the DL extension point keeps its descriptor. - Recreate BertTokenizer in opennlp-api as a thin shim, deprecated since 3.0.0 forRemoval, with the original three Set based constructors and the original tokenizePos message. tokenize delegates to encodeToPieces; ids are synthesized from the set order because the tokenize path never reads them. Null contract follows this branch's reviewed convention, IllegalArgumentException, documented in the throws clauses. - Delete the package-private EncoderTokenizer; the adapter now lives in opennlp-api where downstream code can reach it. AbstractDL's protected createTokenizer returns BertTokenizer again, restoring the override descriptor so an already compiled subclass keeps overriding at runtime, and createPipelineTokenizer hands back the shim. - Delete ReferenceBertPipeline and point the curated and randomized differential tests in WordpieceEncoderTest at the shim, pinning shim and encoder to one sequence. Add BertTokenizerTest covering each constructor's argument validation, the default special token chain, and the exact tokenizePos message. Independent expected sequences continue to live in WordpieceEncoderReferenceSequencesTest. - Fix a real divergence the compatibility check surfaced: the encoder kept U+2028 and U+2029 inside words while the old pipeline split on them, so a word carrying a line or paragraph separator became the unknown piece. cleanAndIsolateCjk now maps Zl and Zp to a space, with a span asserting regression test. - Manual: the WordPiece section describes the deprecation and the migration, including the Set to List vocabulary change. --- .../opennlp/tools/tokenize/BertTokenizer.java | 125 ++++++++++++++++++ .../tools/tokenize/WordpieceEncoder.java | 16 ++- .../src/main/java/opennlp/dl/AbstractDL.java | 37 +++--- .../java/opennlp/dl/EncoderTokenizer.java | 59 --------- .../tools/tokenize/BertTokenizerTest.java | 102 ++++++++++++++ .../tools/tokenize/ReferenceBertPipeline.java | 94 ------------- .../tools/tokenize/WordpieceEncoderTest.java | 35 +++-- opennlp-docs/src/docbkx/tokenizer.xml | 6 +- 8 files changed, 289 insertions(+), 185 deletions(-) create mode 100644 opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java delete mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java delete mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java new file mode 100644 index 0000000000..cea1fd8f53 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/BertTokenizer.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize; + +import java.util.ArrayList; +import java.util.Set; + +import opennlp.tools.util.Span; + +/** + * A {@link Tokenizer} implementation of the full BERT tokenization pipeline: + * basic tokenization (text normalization) followed by wordpiece tokenization, + * with the classification and separator tokens framing every result. + * + * @deprecated Use {@link WordpieceEncoder} instead: + * {@link WordpieceEncoder#encodeToPieces(CharSequence)} returns the same + * {@code String[]} as {@link #tokenize(String)}, and {@code encode} additionally + * carries vocabulary ids and original-text spans. This class is scheduled for + * removal after one stable release. + * + * @see WordpieceEncoder + */ +@Deprecated(since = "3.0.0", forRemoval = true) +public class BertTokenizer implements Tokenizer { + + private final WordpieceEncoder encoder; + + /** + * Initializes a {@link BertTokenizer} for an uncased BERT model, + * with lower casing and accent stripping enabled. + * + * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. + * + * @throws IllegalArgumentException Thrown if the vocabulary is {@code null}, + * contains {@code null}, or is missing a BERT special token. + */ + public BertTokenizer(Set vocabulary) { + this(vocabulary, true); + } + + /** + * Initializes a {@link BertTokenizer} with BERT special tokens. + * + * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. + * @param lowerCase {@code true} for uncased models (lower casing and accent + * stripping), {@code false} for cased models. + * + * @throws IllegalArgumentException Thrown if the vocabulary is {@code null}, + * contains {@code null}, or is missing a BERT special token. + */ + public BertTokenizer(Set vocabulary, boolean lowerCase) { + this(vocabulary, lowerCase, WordpieceTokenizer.BERT_CLS_TOKEN, + WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN); + } + + /** + * Initializes a {@link BertTokenizer} with custom special tokens, for models + * like RoBERTa that do not use the BERT defaults. + * + * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. + * @param lowerCase {@code true} for uncased models (lower casing and + * accent stripping), {@code false} for cased models. + * @param classificationToken The CLS token; must be in the vocabulary. + * @param separatorToken The SEP token; must be in the vocabulary. + * @param unknownToken The UNK token; must be in the vocabulary. + * + * @throws IllegalArgumentException Thrown if any argument is {@code null}, + * the vocabulary contains {@code null}, or a special token is missing + * from the vocabulary. + */ + public BertTokenizer(Set vocabulary, boolean lowerCase, + String classificationToken, String separatorToken, String unknownToken) { + if (vocabulary == null) { + throw new IllegalArgumentException("vocabulary must not be null"); + } + // The encoder assigns each piece its list index as the id. Ids are unused on the + // tokenize() path, so synthesizing them from an arbitrary set order is fine. + this.encoder = new WordpieceEncoder(new ArrayList<>(vocabulary), lowerCase, + classificationToken, separatorToken, unknownToken); + } + + /** + * Tokenizes the given text into wordpieces, surrounded by the classification + * and separator tokens. + * + * @param text The text to tokenize. Must not be {@code null}. + * + * @return The wordpiece tokens. + * + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public String[] tokenize(String text) { + return encoder.encodeToPieces(text); + } + + /** + * Not supported: wordpiece tokens (subwords, {@code ##} continuations and + * special tokens) have no faithful character spans in the original text. + * Use {@link WordpieceEncoder#encode(CharSequence)} for pieces with + * original-text spans. + * + * @throws UnsupportedOperationException Always. + */ + @Override + public Span[] tokenizePos(String text) { + throw new UnsupportedOperationException( + "Wordpiece tokens cannot be mapped to character spans of the original text"); + } + +} diff --git a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java index 49a7910e9c..8a8ed7adbc 100644 --- a/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java +++ b/opennlp-api/src/main/java/opennlp/tools/tokenize/WordpieceEncoder.java @@ -363,7 +363,7 @@ private static MappedText cleanAndIsolateCjk(String original) { i += width; continue; } - if (BertNormalization.isWhitespace(codePoint)) { + if (BertNormalization.isWhitespace(codePoint) || isLineOrParagraphSeparator(codePoint)) { out.add(' ', i, i + width); } else if (BertNormalization.isCjk(codePoint)) { out.add(' ', i, i); @@ -381,6 +381,20 @@ private static MappedText cleanAndIsolateCjk(String original) { return out; } + /** + * A Unicode line or paragraph separator ({@code Zl}, {@code Zp}). These are not whitespace in + * the BERT {@code _is_whitespace} sense, but the reference pipeline's + * {@code whitespace_tokenize} (Python's {@code str.split()}) still breaks words on them, so + * they must become word boundaries here as well. + * + * @param codePoint The code point to classify. + * @return True if the code point is a line or paragraph separator. + */ + private static boolean isLineOrParagraphSeparator(int codePoint) { + final int type = Character.getType(codePoint); + return type == Character.LINE_SEPARATOR || type == Character.PARAGRAPH_SEPARATOR; + } + /** * Isolates punctuation, surrounding each punctuation code point with spaces, preserving the * original-text range of every character. diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java index 01e4d68ca4..8f59c6f844 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/AbstractDL.java @@ -36,8 +36,8 @@ import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import opennlp.tools.tokenize.BertTokenizer; import opennlp.tools.tokenize.Tokenizer; -import opennlp.tools.tokenize.WordpieceEncoder; import opennlp.tools.tokenize.WordpieceTokenizer; import opennlp.tools.util.Span; import opennlp.tools.util.normalizer.AlignedText; @@ -238,20 +238,22 @@ static WordpieceTokenizer createWordpieceTokenizer( } /** - * Creates a {@link Tokenizer} that performs the full BERT tokenization - * pipeline: basic tokenization (text normalization) followed by wordpiece, - * backed by a {@link WordpieceEncoder}. The special tokens are selected - * based on the vocabulary: if it contains RoBERTa-style tokens, those are - * used, otherwise the BERT defaults. + * Creates a {@link BertTokenizer} that performs the full BERT tokenization + * pipeline: basic tokenization (text normalization) followed by wordpiece. + * The special tokens are selected based on the vocabulary: if it contains + * RoBERTa-style tokens, those are used, otherwise the BERT defaults. * * @param vocab The vocabulary map. * @param lowerCase {@code true} for uncased models (lower casing and accent * stripping), {@code false} for cased models. - * @return A configured {@link Tokenizer}. + * @return A configured {@link BertTokenizer}. * @throws IllegalArgumentException Thrown if the selected special tokens * are not all present in the vocabulary. */ - protected Tokenizer createTokenizer( + // The deprecated BertTokenizer stays the return type until its removal in 3.1, so that + // already-compiled subclasses overriding this method keep overriding it. + @SuppressWarnings("removal") + protected BertTokenizer createTokenizer( final Map vocab, final boolean lowerCase) { return createPipelineTokenizer(vocab, lowerCase); @@ -263,29 +265,26 @@ protected Tokenizer createTokenizer( * * @param vocab The vocabulary map. * @param lowerCase {@code true} for uncased models, {@code false} for cased models. - * @return A configured {@link Tokenizer}. + * @return A configured {@link BertTokenizer}. * @throws IllegalArgumentException Thrown if the selected special tokens are not all present in * the vocabulary. */ - static Tokenizer createPipelineTokenizer( + // BertTokenizer is deprecated for removal in 3.1; built here until then. + @SuppressWarnings("removal") + static BertTokenizer createPipelineTokenizer( final Map vocab, final boolean lowerCase) { if (vocab.containsKey( WordpieceTokenizer.ROBERTA_CLS_TOKEN) && vocab.containsKey( WordpieceTokenizer.ROBERTA_SEP_TOKEN)) { - return new EncoderTokenizer(new WordpieceEncoder( - vocab, + return new BertTokenizer( + vocab.keySet(), lowerCase, WordpieceTokenizer.ROBERTA_CLS_TOKEN, WordpieceTokenizer.ROBERTA_SEP_TOKEN, - resolveUnknownToken(vocab))); + resolveUnknownToken(vocab)); } - return new EncoderTokenizer(new WordpieceEncoder( - vocab, - lowerCase, - WordpieceTokenizer.BERT_CLS_TOKEN, - WordpieceTokenizer.BERT_SEP_TOKEN, - WordpieceTokenizer.BERT_UNK_TOKEN)); + return new BertTokenizer(vocab.keySet(), lowerCase); } /** diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java deleted file mode 100644 index 5422c6773d..0000000000 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/EncoderTokenizer.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package opennlp.dl; - -import opennlp.tools.tokenize.Tokenizer; -import opennlp.tools.tokenize.WordpieceEncoder; -import opennlp.tools.util.Span; - -/** - * Adapts a {@link WordpieceEncoder} to the {@link Tokenizer} plumbing of the inference - * classes: {@link #tokenize(String)} returns the encoder's piece strings. - */ -final class EncoderTokenizer implements Tokenizer { - - private final WordpieceEncoder encoder; - - /** - * Instantiates the adapter. - * - * @param encoder The encoder whose pieces this tokenizer returns. - */ - EncoderTokenizer(final WordpieceEncoder encoder) { - this.encoder = encoder; - } - - /** {@inheritDoc} */ - @Override - public String[] tokenize(final String text) { - return encoder.encodeToPieces(text); - } - - /** - * Not supported under the {@link Tokenizer} contract, whose spans are expected to contain - * their token's surface form; wordpiece pieces are not substrings of the input. Use - * {@link WordpieceEncoder#encode(CharSequence)} for pieces with original-text spans. - * - * @throws UnsupportedOperationException Always. - */ - @Override - public Span[] tokenizePos(final String text) { - throw new UnsupportedOperationException( - "Wordpiece tokens cannot be mapped to character spans of the original text"); - } -} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java new file mode 100644 index 0000000000..070834316b --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/BertTokenizerTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.tokenize; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Pins the deprecated {@link BertTokenizer} shim: constructor validation, delegation to + * {@link WordpieceEncoder#encodeToPieces(CharSequence)}, the default special tokens, and + * the unsupported {@link BertTokenizer#tokenizePos(String)}. + */ +@SuppressWarnings("removal") // Exercises BertTokenizer deliberately until its removal in 3.1. +class BertTokenizerTest { + + private static final List VOCABULARY = List.of( + "[PAD]", "[UNK]", "[CLS]", "[SEP]", "hello", "world", "##s", "ca", "##fe", ",", "!"); + + private static Set vocabularySet() { + return new HashSet<>(VOCABULARY); + } + + @ParameterizedTest + @ValueSource(strings = {"Hello, world!", "Caf\u00E9 hellos", ""}) + void testTokenizeReturnsTheEncoderPieceSequence(String input) { + final BertTokenizer tokenizer = new BertTokenizer(vocabularySet()); + final WordpieceEncoder encoder = new WordpieceEncoder(VOCABULARY); + assertArrayEquals(encoder.encodeToPieces(input), tokenizer.tokenize(input), + "delegation broke on: " + input); + } + + @Test + void testDefaultConstructorsChainToBertSpecialTokensAndLowerCasing() { + final String[] uncasedDefault = new BertTokenizer(vocabularySet()).tokenize("Hello worldS"); + assertArrayEquals(new String[] {"[CLS]", "hello", "world", "##s", "[SEP]"}, uncasedDefault); + assertArrayEquals(uncasedDefault, + new BertTokenizer(vocabularySet(), true).tokenize("Hello worldS")); + assertArrayEquals(uncasedDefault, + new BertTokenizer(vocabularySet(), true, WordpieceTokenizer.BERT_CLS_TOKEN, + WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN) + .tokenize("Hello worldS")); + } + + @Test + void testCasedTokenizerKeepsCase() { + // Without lower casing, the capitalized word misses the lowercase-only vocabulary. + assertArrayEquals(new String[] {"[CLS]", "[UNK]", "[SEP]"}, + new BertTokenizer(vocabularySet(), false).tokenize("Hello")); + } + + @Test + void testConstructorsRejectNullArguments() { + assertThrows(IllegalArgumentException.class, () -> new BertTokenizer(null)); + assertThrows(IllegalArgumentException.class, () -> new BertTokenizer(null, true)); + assertThrows(IllegalArgumentException.class, + () -> new BertTokenizer(null, true, "[CLS]", "[SEP]", "[UNK]")); + assertThrows(IllegalArgumentException.class, + () -> new BertTokenizer(vocabularySet(), true, null, "[SEP]", "[UNK]")); + assertThrows(IllegalArgumentException.class, + () -> new BertTokenizer(vocabularySet(), true, "[CLS]", null, "[UNK]")); + assertThrows(IllegalArgumentException.class, + () -> new BertTokenizer(vocabularySet(), true, "[CLS]", "[SEP]", null)); + } + + @Test + void testTokenizeRejectsNullText() { + final BertTokenizer tokenizer = new BertTokenizer(vocabularySet()); + assertThrows(IllegalArgumentException.class, () -> tokenizer.tokenize(null)); + } + + @Test + void testTokenizePosIsUnsupportedWithTheDocumentedMessage() { + final BertTokenizer tokenizer = new BertTokenizer(vocabularySet()); + final UnsupportedOperationException e = assertThrows(UnsupportedOperationException.class, + () -> tokenizer.tokenizePos("hello world")); + assertEquals("Wordpiece tokens cannot be mapped to character spans of the original text", + e.getMessage()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java deleted file mode 100644 index ca7b2b5618..0000000000 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/ReferenceBertPipeline.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package opennlp.tools.tokenize; - -import java.text.Normalizer; -import java.util.Locale; -import java.util.Set; - -/** - * The reference BERT basic-tokenization stage feeding {@link WordpieceTokenizer}, serving as - * the differential baseline for {@link WordpieceEncoderTest}: the encoder's piece sequence - * must match this pipeline exactly. - */ -final class ReferenceBertPipeline { - - private static final int MAX_WORD_CHARACTERS = 100; - - private final WordpieceTokenizer wordpieceTokenizer; - private final boolean lowerCase; - - ReferenceBertPipeline(Set vocabulary, boolean lowerCase) { - this.wordpieceTokenizer = new WordpieceTokenizer(vocabulary, - WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, - WordpieceTokenizer.BERT_UNK_TOKEN, MAX_WORD_CHARACTERS); - this.lowerCase = lowerCase; - } - - String[] tokenize(String text) { - return wordpieceTokenizer.tokenize(normalize(text)); - } - - private String normalize(String text) { - String normalized = cleanText(text); - normalized = isolateCjkCharacters(normalized); - if (lowerCase) { - // Locale.ROOT lower casing is the reference behavior of BERT's do_lower_case: the full - // locale-independent Unicode case mappings, including one-to-many ones. - normalized = stripAccents(normalized.toLowerCase(Locale.ROOT)); - } - return BertNormalization.isolatePunctuation(normalized); - } - - private static String cleanText(String text) { - final StringBuilder cleaned = new StringBuilder(text.length()); - text.codePoints().forEach(codePoint -> { - if (codePoint == 0 || codePoint == 0xFFFD || BertNormalization.isControl(codePoint)) { - return; - } - if (BertNormalization.isWhitespace(codePoint)) { - cleaned.append(' '); - } else { - cleaned.appendCodePoint(codePoint); - } - }); - return cleaned.toString(); - } - - private static String isolateCjkCharacters(String text) { - final StringBuilder spaced = new StringBuilder(text.length()); - text.codePoints().forEach(codePoint -> { - if (BertNormalization.isCjk(codePoint)) { - spaced.append(' ').appendCodePoint(codePoint).append(' '); - } else { - spaced.appendCodePoint(codePoint); - } - }); - return spaced.toString(); - } - - private static String stripAccents(String text) { - final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); - final StringBuilder stripped = new StringBuilder(decomposed.length()); - decomposed.codePoints().forEach(codePoint -> { - if (Character.getType(codePoint) != Character.NON_SPACING_MARK) { - stripped.appendCodePoint(codePoint); - } - }); - return stripped.toString(); - } -} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java index 0cc97d3b6c..997036b092 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderTest.java @@ -35,8 +35,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * The encoder held against {@link ReferenceBertPipeline} for piece-sequence parity (the encoder's - * contract is "the same pipeline, plus ids and spans"), plus exact hand-computed span + * The encoder held against the deprecated {@link BertTokenizer} for piece-sequence parity (the + * shim's contract is "the encoder's piece sequence, nothing more"), plus exact hand-computed span * assertions through every normalization step that changes, inserts, or removes characters. */ class WordpieceEncoderTest { @@ -95,21 +95,22 @@ static Stream curatedInputs() { @ParameterizedTest @MethodSource("curatedInputs") - void testPieceSequenceMatchesTheReferencePipelineOnCuratedInputs(String input) { - final ReferenceBertPipeline reference = new ReferenceBertPipeline(new HashSet<>(VOCAB), true); + @SuppressWarnings("removal") // BertTokenizer is pinned to the encoder until its removal in 3.1. + void testPieceSequenceMatchesBertTokenizerOnCuratedInputs(String input) { + final BertTokenizer bertTokenizer = new BertTokenizer(new HashSet<>(VOCAB), true); final WordpieceEncoder encoder = uncased(); - assertArrayEquals(reference.tokenize(input), encoder.encodeToPieces(input), + assertArrayEquals(bertTokenizer.tokenize(input), encoder.encodeToPieces(input), "parity broke on: " + input); } @Test - void testPieceSequenceMatchesTheReferencePipelineOnRandomInputs() { + @SuppressWarnings("removal") // BertTokenizer is pinned to the encoder until its removal in 3.1. + void testPieceSequenceMatchesBertTokenizerOnRandomInputs() { final int[] pool = {'a', 'b', 'A', 'B', 'z', ' ', ' ', '\t', 0x00A0, '.', '!', ',', 0x0301, 0x00E9, 0x0130, 0x03A3, 0x03C3, 0x03BF, 0x4E2D, 0xFFFD, 0x200B, 0x1F600, 0}; final Random random = new Random(42); for (final boolean lowerCase : new boolean[] {true, false}) { - final ReferenceBertPipeline reference = - new ReferenceBertPipeline(new HashSet<>(VOCAB), lowerCase); + final BertTokenizer bertTokenizer = new BertTokenizer(new HashSet<>(VOCAB), lowerCase); final WordpieceEncoder encoder = new WordpieceEncoder(VOCAB, lowerCase); for (int round = 0; round < 400; round++) { final StringBuilder text = new StringBuilder(); @@ -118,7 +119,7 @@ void testPieceSequenceMatchesTheReferencePipelineOnRandomInputs() { text.appendCodePoint(pool[random.nextInt(pool.length)]); } final String input = text.toString(); - assertArrayEquals(reference.tokenize(input), encoder.encodeToPieces(input), + assertArrayEquals(bertTokenizer.tokenize(input), encoder.encodeToPieces(input), "parity broke on: " + input); // Span invariants: within bounds and never moving backwards. @@ -170,6 +171,18 @@ void testCjkIsolationYieldsOnePieceAndSpanPerIdeograph() { assertPiece(pieces.get(2), "\u56FD", 14, 1, 2); } + @Test + void testLineAndParagraphSeparatorsSplitWords() { + // Zl and Zp are not whitespace in the BERT _is_whitespace sense, but the reference + // pipeline's whitespace_tokenize (Python's str.split()) breaks words on them, as did the + // previous OpenNLP pipeline via WhitespaceTokenizer. + final List pieces = uncased().encode("hello\u2028world\u2029hello"); + assertEquals(5, pieces.size()); + assertPiece(pieces.get(1), "hello", 4, 0, 5); + assertPiece(pieces.get(2), "world", 5, 6, 11); + assertPiece(pieces.get(3), "hello", 4, 12, 17); + } + @Test void testUnknownWordCoversItsWholeSurfaceIncludingRemovedChars() { // NUL and the zero-width space are removed by cleaning, so one word "abc" remains; it is @@ -182,8 +195,8 @@ void testUnknownWordCoversItsWholeSurfaceIncludingRemovedChars() { @Test void testContextualCaseMappingFallsBackToWordWideSpans() { // Greek final sigma is a contextual mapping the per-char rerun cannot reproduce, so the - // word's pieces fall back to spanning the whole word; content parity is asserted in the - // differential tests above. + // word's pieces fall back to spanning the whole word; the piece content is asserted + // exactly below. final List pieces = uncased().encode("\u03A3\u039F\u03A6\u039F\u03A3"); assertEquals(3, pieces.size()); diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index 0c84b5c855..c3906cd144 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -612,7 +612,11 @@ int[] ids = encoder.encodeToIds("OpenNLP encodes text for BERT models.");]]> spans at the text's boundaries, and words the vocabulary cannot cover become the unknown piece. Models with other special tokens or with non-contiguous ids are supported through the constructors taking explicit special tokens or a piece-to-id - map. + map. The older BertTokenizer is deprecated and scheduled for removal: + its tokenize method returns the same piece sequence that + encodeToPieces returns. Migrating means passing the vocabulary as a + List or a piece-to-id map instead of a Set; the output + is unchanged.
From 632d996130abefc95551a4f0b46010a56e939d93 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 08:30:53 -0400 Subject: [PATCH 18/82] OPENNLP-1885: Document fixture regeneration and Utf8Text span mapping Add a README for regenerating the bundled SentencePiece parity fixtures and clarify that Utf8Text is the encode-path span bridge behind SubwordPiece offsets. --- .../subword/sentencepiece/Utf8Text.java | 3 +- .../opennlp/subword/sentencepiece/README.md | 71 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java index 4f2bd95f0f..149b45be1e 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/Utf8Text.java @@ -22,7 +22,8 @@ * *

The pipeline runs in UTF-8 byte space, but the spans reported to the caller must be UTF-16 * offsets into the original {@code CharSequence}; this map converts them. An unpaired surrogate, - * which UTF-8 cannot represent, is encoded as U+FFFD.

+ * which UTF-8 cannot represent, is encoded as U+FFFD. Callers see those offsets only through + * {@link opennlp.tools.tokenize.SubwordPiece} spans.

*/ final class Utf8Text { diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md new file mode 100644 index 0000000000..5c39192796 --- /dev/null +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md @@ -0,0 +1,71 @@ + + +# SentencePiece parity fixtures + +Tiny trained `.model` files and matching `.fixtures.tsv` files used by +`SentencePieceParityTest` and related tests. They are **not** third-party +pretrained models: they are generated in-tree from `corpus.txt` plus a short +multilingual add-on list in `gen_fixtures.py`, using the reference +[sentencepiece](https://github.com/google/sentencepiece) Python package. + +## Regenerating the tiny models + +From this directory (or any directory; pass absolute paths as needed): + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install sentencepiece +python gen_fixtures.py corpus.txt . +``` + +That trains each model listed in `MODELS` inside `gen_fixtures.py` +(`tiny-unigram`, `tiny-unigram-bytefb`, `tiny-bpe`, `tiny-unigram-identity`, +`tiny-unigram-suffix`) and writes: + +- `.model` — SentencePiece binary model +- `.fixtures.tsv` — expected pieces, ids, UTF-16 spans, and normalized form +- `corpus-full.txt` — training corpus (`corpus.txt` plus multilingual lines) + +Pin the `sentencepiece` package version you used if regenerating for a PR, so +reviewers can reproduce the same bytes. + +## Real-model fixtures (optional, not bundled) + +`gen_real_fixtures.py` writes the same TSV format for any directory of +pre-trained `*.model` files (no training). It reuses the escaping helpers and +input list from `gen_fixtures.py`: + +```bash +source .venv/bin/activate # same venv as above +python gen_real_fixtures.py /path/to/models +``` + +Real models and their TSVs are not checked into this tree; the script is for +local eval against published SentencePiece models. + +## Fixture TSV format + +Tab-separated, with backslash escapes (`\\`, `\t`, `\n`, `\r`): + +```text +esc(input) TAB pieceCount TAB [esc(piece) TAB id TAB begin TAB end]... TAB esc(normalized) +``` + +`begin` / `end` are UTF-16 code-unit offsets into the original input (Java +`String` indexing). From 9670c34d566fc6adb47dfcdd0a423d8743af7c20 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 12:43:37 -0400 Subject: [PATCH 19/82] OPENNLP-1885: Attribute the double-array literature in DoubleArrayTrie State that the reader is an independent re-implementation of the serialized format, cite Aoe (1989), Yata et al. (2007), and Kanda et al. (2023) in the class javadoc, and decode the bit-9 offset extension with a plain conditional instead of the branchless form. --- .../subword/sentencepiece/DoubleArrayTrie.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java index 75f48e1de8..5bed8a6852 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/DoubleArrayTrie.java @@ -28,7 +28,19 @@ * state. Out-of-range unit references, which a well-formed trie never produces, fail loudly * rather than reading arbitrary memory.

* + *

This class is an independent re-implementation of the reader side of that format, written + * against the published double-array literature: the trie itself is Aoe's double-array, the unit + * encoding is the compact static variant of Yata et al., and the bit-9 offset extension is the + * two-kinds-of-offset scheme described by Kanda et al. Darts-clone, by the same Yata, is the + * reference implementation of the format.

+ * * @see Darts-clone + * @see Aoe (1989): An Efficient Digital Search + * Algorithm by Using a Double-Array Structure + * @see Yata et al. (2007): A compact static + * double-array keeping character codes + * @see Kanda et al. (2023): Engineering faster + * double-array Aho-Corasick automata */ final class DoubleArrayTrie implements Serializable { @@ -127,13 +139,12 @@ boolean hasTransitionFromRoot(int b) { /** * Returns the offset from a unit to its children, as encoded by Darts-clone: bits 10 to 30 hold * the raw offset, and bit 9 is an extension flag that scales it by 256 for far-away children. - * The expression {@code (unit & (1 << 9)) >>> 6} evaluates to 8 exactly when bit 9 is set, so - * the raw offset is shifted left by either 0 or 8 bits. * * @param unit The unit word. * @return The child offset. */ private static int offset(int unit) { - return (unit >>> 10) << ((unit & (1 << 9)) >>> 6); + final int raw = unit >>> 10; + return (unit & 1 << 9) == 0 ? raw : raw << 8; } } From d8b04491ae2a47408960119e752331c549c08636 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 13:23:32 -0400 Subject: [PATCH 20/82] OPENNLP-1885: Expand fixtures README into a validation tutorial and link it from the manual State that the reference implementation produces the expected fixture outputs, add the end-to-end validation steps for the bundled and real models, and point the manual's SentencePiece section at the README. --- opennlp-docs/src/docbkx/tokenizer.xml | 9 ++++++ .../opennlp/subword/sentencepiece/README.md | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index c3906cd144..03776b9404 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -589,6 +589,15 @@ int[] ids = tokenizer.encodeToIds("Ready for the embedding layer.");]]> in . This is useful when other processing must see text exactly as the subword model does. + + Output parity with the reference implementation is pinned by test fixtures whose + expected pieces, ids, and spans are produced by the reference + sentencepiece + package rather than by the Java code under test. The + fixtures README + documents how the bundled test models were created and walks through regenerating + them and validating the implementation, including against real published models. +
WordPiece diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md index 5c39192796..6b8cbc39db 100644 --- a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md +++ b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md @@ -23,6 +23,10 @@ pretrained models: they are generated in-tree from `corpus.txt` plus a short multilingual add-on list in `gen_fixtures.py`, using the reference [sentencepiece](https://github.com/google/sentencepiece) Python package. +The expected outputs in the TSVs come from the reference implementation, not +from the Java code under test, so the parity tests stay independent of the +implementation they check. + ## Regenerating the tiny models From this directory (or any directory; pass absolute paths as needed): @@ -45,6 +49,31 @@ That trains each model listed in `MODELS` inside `gen_fixtures.py` Pin the `sentencepiece` package version you used if regenerating for a PR, so reviewers can reproduce the same bytes. +## Validating the Java implementation + +To verify parity end to end, regenerate the fixtures as above, then run the +test suite from the repository root: + +```bash +./mvnw -pl opennlp-extensions/opennlp-subword -am test +``` + +`SentencePieceParityTest` asserts every fixture line piece for piece, span +for span, against `SentencePieceTokenizer`, for each bundled tiny model. + +To additionally validate against real published models, generate fixtures for +a directory of pre-trained `*.model` files and point the eval test at it: + +```bash +source .venv/bin/activate +python gen_real_fixtures.py /path/to/models +./mvnw -pl opennlp-extensions/opennlp-subword -am test \ + -Dopennlp.subword.eval.dir=/path/to/models +``` + +`SentencePieceRealModelEvalTest` is skipped unless +`opennlp.subword.eval.dir` is set. + ## Real-model fixtures (optional, not bundled) `gen_real_fixtures.py` writes the same TSV format for any directory of From dcc33bd68321f6cd40f20eea033b9bad3a9f1f04 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:40:42 -0400 Subject: [PATCH 21/82] OPENNLP-1885: Pin InvalidFormatException on a malformed precompiled character map --- .../SentencePieceModelValidationTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java index a207625290..5f340eda3a 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java @@ -88,6 +88,25 @@ void testMissingUnknownPieceFailsLoudly() { assertTrue(e.getMessage().contains("unknown piece"), e.getMessage()); } + @Test + void testMalformedPrecompiledCharsMapFailsLoudly() { + // A well-formed proto whose normalizer spec carries a truncated precompiled character map; + // load() must report it as an invalid model, like every other malformed model content. + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + writePiece(out, "", 2); + writePiece(out, "a", 1); + // normalizer_spec { precompiled_charsmap = <3 bytes> } + out.write(0x1A); + out.write(5); + out.write(0x12); + out.write(3); + out.writeBytes(new byte[] {1, 2, 3}); + final byte[] model = out.toByteArray(); + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> SentencePieceTokenizer.load(new ByteArrayInputStream(model))); + assertTrue(e.getMessage().contains("character map"), e.getMessage()); + } + @Test void testConcurrentEncodingIsConsistent() throws Exception { final SentencePieceTokenizer tokenizer = SentencePieceFixtures.tokenizer("tiny-unigram"); From f512cece482525360353908e0145b90396e248d0 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:43:46 -0400 Subject: [PATCH 22/82] OPENNLP-1885: Fix loader exception type and wire opennlp-subword into distr --- .../WordpieceEncoderReferenceSequencesTest.java | 2 +- opennlp-distr/pom.xml | 4 ++++ opennlp-distr/src/main/assembly/bin.xml | 7 +++++++ .../opennlp/subword/sentencepiece/BpeEncoder.java | 11 ++++++----- .../subword/sentencepiece/ModelProtoReader.java | 3 --- .../sentencepiece/SentencePieceNormalizer.java | 15 +++++++++------ .../sentencepiece/SentencePieceTokenizer.java | 3 ++- .../SentencePieceUsageExampleTest.java | 2 +- opennlp-extensions/pom.xml | 2 +- pom.xml | 6 ++++++ 10 files changed, 37 insertions(+), 18 deletions(-) diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java index 064d644a57..bbaeadaca3 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/tokenize/WordpieceEncoderReferenceSequencesTest.java @@ -36,7 +36,7 @@ * present in the vocabulary (every piece must have an id), so the vocabularies * here include them; the token sequences are unchanged. */ -public class WordpieceEncoderReferenceSequencesTest { +class WordpieceEncoderReferenceSequencesTest { private static final List VOCABULARY = List.of( "[CLS]", "[SEP]", "[UNK]", diff --git a/opennlp-distr/pom.xml b/opennlp-distr/pom.xml index e9092d8821..7e8501bca9 100644 --- a/opennlp-distr/pom.xml +++ b/opennlp-distr/pom.xml @@ -91,6 +91,10 @@ org.apache.opennlp opennlp-spellcheck + + org.apache.opennlp + opennlp-subword + diff --git a/opennlp-distr/src/main/assembly/bin.xml b/opennlp-distr/src/main/assembly/bin.xml index 2db4eafc65..b861de7caf 100644 --- a/opennlp-distr/src/main/assembly/bin.xml +++ b/opennlp-distr/src/main/assembly/bin.xml @@ -239,6 +239,13 @@ docs/apidocs/opennlp-spellcheck + + ../opennlp-extensions/opennlp-subword/target/reports/apidocs + 644 + 755 + docs/apidocs/opennlp-subword + + ../opennlp-extensions/opennlp-uima/target/reports/apidocs 644 diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java index 6e448e5c38..1c813d5314 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/BpeEncoder.java @@ -91,10 +91,10 @@ List encode(byte[] normalized, int size) { } // The symbol list as index-linked ranges of the normalized bytes; merged-away symbols - // become empty ranges. + // become empty ranges. Freeze flags travel as 0/1 bytes parallel to the ranges. final IntBuilder fromB = new IntBuilder(size); final IntBuilder toB = new IntBuilder(size); - final List freezeList = new ArrayList<>(); + final ByteBuilder freezeB = new ByteBuilder(size); int position = 0; while (position < size) { int matched = 0; @@ -107,19 +107,20 @@ List encode(byte[] normalized, int size) { size - position); fromB.append(position); toB.append(position + length); - freezeList.add(frozen); + freezeB.append(frozen ? (byte) 1 : (byte) 0); position += length; } - final int symbolCount = freezeList.size(); + final int symbolCount = freezeB.length(); final int[] from = fromB.toArray(); final int[] to = toB.toArray(); + final byte[] frozenFlags = freezeB.array(); final int[] prev = new int[symbolCount]; final int[] next = new int[symbolCount]; final boolean[] freeze = new boolean[symbolCount]; for (int i = 0; i < symbolCount; i++) { prev[i] = i - 1; next[i] = i + 1 < symbolCount ? i + 1 : -1; - freeze[i] = freezeList.get(i); + freeze[i] = frozenFlags[i] != 0; } // Higher score first; equal scores break towards the leftmost pair. diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java index a6ceca6b84..e9f4cb0bbd 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/ModelProtoReader.java @@ -58,7 +58,6 @@ final class ModelProtoReader { private static final int FIELD_TRAINER_MODEL_TYPE = 3; private static final int FIELD_TRAINER_TREAT_WHITESPACE_AS_SUFFIX = 24; private static final int FIELD_TRAINER_BYTE_FALLBACK = 35; - private static final int FIELD_TRAINER_UNK_ID = 40; // Field numbers of the NormalizerSpec sub-message. private static final int FIELD_NORMALIZER_PRECOMPILED_CHARSMAP = 2; @@ -182,7 +181,6 @@ private void trainerSpec(RawModel model, int end) throws InvalidFormatException case FIELD_TRAINER_TREAT_WHITESPACE_AS_SUFFIX -> model.treatWhitespaceAsSuffix = varintOf(tag) != 0; case FIELD_TRAINER_BYTE_FALLBACK -> model.byteFallback = varintOf(tag) != 0; - case FIELD_TRAINER_UNK_ID -> model.unkId = (int) varintOf(tag); default -> skip(tag); } } @@ -406,7 +404,6 @@ static final class RawModel { int modelType = MODEL_TYPE_UNIGRAM; boolean byteFallback = false; boolean treatWhitespaceAsSuffix = false; - int unkId = 0; byte[] precompiledCharsMap = new byte[0]; boolean addDummyPrefix = true; diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java index 5494282e95..11ded3443c 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceNormalizer.java @@ -18,6 +18,8 @@ import java.io.Serializable; +import opennlp.tools.util.InvalidFormatException; + /** * The model-embedded text normalizer of a SentencePiece model, operating in UTF-8 byte space. * @@ -61,11 +63,12 @@ final class SentencePieceNormalizer implements Serializable { * @param userDefinedMatcher Longest-match trie over user-defined symbols that must pass * through normalization untouched, or null when the model * defines none. - * @throws IllegalArgumentException Thrown if the character map is structurally invalid. + * @throws InvalidFormatException Thrown if the character map is structurally invalid. */ SentencePieceNormalizer(byte[] precompiledCharsMap, boolean addDummyPrefix, boolean removeExtraWhitespaces, boolean escapeWhitespaces, - boolean treatWhitespaceAsSuffix, PieceTrie userDefinedMatcher) { + boolean treatWhitespaceAsSuffix, PieceTrie userDefinedMatcher) + throws InvalidFormatException { if (precompiledCharsMap.length == 0) { trie = null; blob = null; @@ -73,24 +76,24 @@ final class SentencePieceNormalizer implements Serializable { } else { // Layout: . if (precompiledCharsMap.length <= 4) { - throw new IllegalArgumentException("The precompiled character map is truncated."); + throw new InvalidFormatException("The precompiled character map is truncated."); } final long trieSize = (precompiledCharsMap[0] & 0xFFL) | (precompiledCharsMap[1] & 0xFFL) << 8 | (precompiledCharsMap[2] & 0xFFL) << 16 | (precompiledCharsMap[3] & 0xFFL) << 24; if (trieSize >= precompiledCharsMap.length - 4) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "The precompiled character map declares a trie of " + trieSize + " bytes but only " + (precompiledCharsMap.length - 4) + " bytes follow."); } if (trieSize < 1024 || (trieSize & 0x3FF) != 0) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "The precompiled character map trie size " + trieSize + " is not a positive multiple of 1024."); } if (precompiledCharsMap[precompiledCharsMap.length - 1] != 0) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "The precompiled character map replacement block is not null-terminated."); } trie = new DoubleArrayTrie(precompiledCharsMap, 4, (int) trieSize); diff --git a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java index 041cd7b285..74caac0683 100644 --- a/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java +++ b/opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java @@ -140,7 +140,7 @@ private SentencePieceTokenizer(ModelProtoReader.RawModel model) throws InvalidFo for (int i = 0; i < count; i++) { final String piece = pieces[i]; if (piece.length() >= MAX_PIECE_LENGTH) { - throw new InvalidFormatException("The piece with id " + i + " is longer than " + throw new InvalidFormatException("The piece with id " + i + " must be shorter than " + MAX_PIECE_LENGTH + " characters."); } if (piece.indexOf(0) >= 0) { @@ -651,6 +651,7 @@ public String idToPiece(int id) { * * @param piece The piece to look up; must not be null. * @return The id, or the unknown id when the vocabulary does not contain the piece. + * @throws IllegalArgumentException Thrown if {@code piece} is null. */ public int pieceToId(String piece) { if (piece == null) { diff --git a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java index 33c375574b..b1bd4cc9ee 100644 --- a/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java +++ b/opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceUsageExampleTest.java @@ -37,7 +37,7 @@ * {@link SentencePieceTokenizer} from a {@code .model} file, encode text to pieces with * original offsets, and obtain id arrays. */ -public class SentencePieceUsageExampleTest { +class SentencePieceUsageExampleTest { @Test void testLoadEncodeAndEncodeToIds(@TempDir Path dir) throws IOException { diff --git a/opennlp-extensions/pom.xml b/opennlp-extensions/pom.xml index dd233553e0..2b0e87273d 100644 --- a/opennlp-extensions/pom.xml +++ b/opennlp-extensions/pom.xml @@ -39,8 +39,8 @@ opennlp-morfologik - opennlp-subword opennlp-spellcheck + opennlp-subword opennlp-uima diff --git a/pom.xml b/pom.xml index 53aa93d56d..3f77763325 100644 --- a/pom.xml +++ b/pom.xml @@ -216,6 +216,12 @@ ${project.version} + + opennlp-subword + ${project.groupId} + ${project.version} + + opennlp-uima ${project.groupId} From 45db4212dbabf8b2b7f35d02afdf322ee42168bd Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:20:14 -0400 Subject: [PATCH 23/82] OPENNLP-1885: Reference the fixtures README by its in-tree path The absolute GitHub URL 404s until merge and pins the branch layout. --- opennlp-docs/src/docbkx/tokenizer.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennlp-docs/src/docbkx/tokenizer.xml b/opennlp-docs/src/docbkx/tokenizer.xml index 03776b9404..6733728d3a 100644 --- a/opennlp-docs/src/docbkx/tokenizer.xml +++ b/opennlp-docs/src/docbkx/tokenizer.xml @@ -593,8 +593,8 @@ int[] ids = tokenizer.encodeToIds("Ready for the embedding layer.");]]> Output parity with the reference implementation is pinned by test fixtures whose expected pieces, ids, and spans are produced by the reference sentencepiece - package rather than by the Java code under test. The - fixtures README + package rather than by the Java code under test. The fixtures README at + opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/README.md documents how the bundled test models were created and walks through regenerating them and validating the implementation, including against real published models. From 2d9588791a4b7c490c452f5db823fe1ad7d2b254 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 09:23:40 -0400 Subject: [PATCH 24/82] Add opennlp-embeddings module with a safetensors reader New extension module, targeting a modern (2025) static-embedding distillation format as OpenNLP's word2vec/GloVe successor: same flat per-token vector table artifact shape, pure JVM lookup at inference time, no PyTorch/ONNX runtime dependency. SafetensorsFile/SafetensorsHeaderParser read the safetensors format (8-byte little-endian header length, JSON header describing each tensor's dtype/shape/byte range, then raw tensor bytes). Hand-rolled cursor parser scoped to the header's actual shape, no third-party JSON dependency, matching the project's existing data-file reader discipline. safetensors carries no executable content (unlike PyTorch's pickle-based checkpoints), so no XXE-style hardening is needed, only ordinary malformed-input handling. singleMatrixTensorName() deliberately does not guess a tensor key name convention: distillation tools do not agree on one, so it auto-detects the lone 2-D F32 tensor and fails loud listing every candidate when that is ambiguous, rather than risk silently loading the wrong tensor. Next: tokenizer wiring and the mean-pool/normalize lookup path, targeting minishlab/potion-base-8M as the v1 reference model. --- opennlp-extensions/opennlp-embeddings/pom.xml | 64 +++ .../opennlp/embeddings/SafetensorsFile.java | 211 ++++++++++ .../embeddings/SafetensorsHeaderParser.java | 392 ++++++++++++++++++ .../java/opennlp/embeddings/TensorInfo.java | 45 ++ .../embeddings/SafetensorsFileTest.java | 261 ++++++++++++ opennlp-extensions/pom.xml | 1 + 6 files changed, 974 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/pom.xml create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java diff --git a/opennlp-extensions/opennlp-embeddings/pom.xml b/opennlp-extensions/opennlp-embeddings/pom.xml new file mode 100644 index 0000000000..d1a724d80f --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/pom.xml @@ -0,0 +1,64 @@ + + + + + + 4.0.0 + + org.apache.opennlp + opennlp-extensions + 3.0.0-SNAPSHOT + + + opennlp-embeddings + jar + Apache OpenNLP :: Ext :: Embeddings + + + + org.apache.opennlp + opennlp-api + + + + org.apache.opennlp + opennlp-runtime + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java new file mode 100644 index 0000000000..3e1bad57e3 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Reads a safetensors file: an 8-byte + * little-endian header length, a JSON header describing each tensor's dtype, shape, and byte + * range, followed by the raw tensor bytes. Deliberately not a general tensor-format library: + * only the {@code F32} decode path {@link #readFloat32(String)} needs is implemented, since + * that is what a distilled static-embedding table stores. + * + *

Security. Unlike PyTorch's pickle-based checkpoint format, safetensors carries no + * executable content: the header is data-only JSON and the body is raw tensor bytes, so loading + * one cannot execute arbitrary code. No hardening beyond ordinary malformed-input handling is + * needed.

+ * + *

The whole file is read into memory up front (matching the project's existing bundled-data + * readers), which is appropriate for the small (tens of megabytes) tables this module targets. + * Instances are immutable and safe for concurrent reads after construction.

+ */ +public final class SafetensorsFile { + + private static final int HEADER_LENGTH_PREFIX_BYTES = 8; + + private final byte[] bytes; + private final long dataStart; + private final Map tensorsByName; + private final Map metadata; + + private SafetensorsFile(byte[] bytes, long dataStart, Map tensorsByName, + Map metadata) { + this.bytes = bytes; + this.dataStart = dataStart; + this.tensorsByName = tensorsByName; + this.metadata = metadata; + } + + /** + * Reads a safetensors file. + * + * @param file The file to read. Must not be {@code null} and must exist. + * @return The parsed file, with every tensor's metadata resolved and validated against the + * file's actual length. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing, or the + * file is malformed. + * @throws UncheckedIOException Thrown if reading the file fails. + */ + public static SafetensorsFile read(Path file) { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); + } + final byte[] bytes; + try { + bytes = Files.readAllBytes(file); + } + catch (IOException e) { + throw new UncheckedIOException("Unable to read safetensors file " + file, e); + } + if (bytes.length < HEADER_LENGTH_PREFIX_BYTES) { + throw new IllegalArgumentException( + "File " + file + " is too short to be a safetensors file: " + bytes.length + " bytes"); + } + final long headerLength = ByteBuffer.wrap(bytes, 0, HEADER_LENGTH_PREFIX_BYTES) + .order(ByteOrder.LITTLE_ENDIAN).getLong(); + final long dataStart = (long) HEADER_LENGTH_PREFIX_BYTES + headerLength; + if (headerLength < 0 || dataStart > bytes.length) { + throw new IllegalArgumentException("File " + file + " declares a header length of " + + headerLength + ", which does not fit in a file of " + bytes.length + " bytes"); + } + final String headerJson = new String(bytes, HEADER_LENGTH_PREFIX_BYTES, (int) headerLength, + StandardCharsets.UTF_8); + final SafetensorsHeaderParser.Result parsed = SafetensorsHeaderParser.parse(headerJson); + final Map tensorsByName = new LinkedHashMap<>(parsed.tensors().size() * 2); + for (final TensorInfo tensor : parsed.tensors()) { + if (tensor.dataOffsetBegin() < 0 || tensor.dataOffsetEnd() < tensor.dataOffsetBegin() + || dataStart + tensor.dataOffsetEnd() > bytes.length) { + throw new IllegalArgumentException("File " + file + " tensor '" + tensor.name() + + "' has a data range [" + tensor.dataOffsetBegin() + ", " + tensor.dataOffsetEnd() + + ") that does not fit in the file"); + } + if (tensorsByName.putIfAbsent(tensor.name(), tensor) != null) { + throw new IllegalArgumentException( + "File " + file + " declares tensor '" + tensor.name() + "' more than once"); + } + } + return new SafetensorsFile(bytes, dataStart, Collections.unmodifiableMap(tensorsByName), + Collections.unmodifiableMap(parsed.metadata())); + } + + /** {@return the names of every tensor declared in the header, in header order} */ + public Set tensorNames() { + return tensorsByName.keySet(); + } + + /** + * Returns the header metadata for one tensor. + * + * @param name The tensor's name. Must not be {@code null}. + * @return The tensor's metadata. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null} or not a tensor in + * this file. + */ + public TensorInfo tensorInfo(String name) { + if (name == null) { + throw new IllegalArgumentException("Name must not be null"); + } + final TensorInfo info = tensorsByName.get(name); + if (info == null) { + throw new IllegalArgumentException( + "No tensor named '" + name + "' in this file; available: " + tensorsByName.keySet()); + } + return info; + } + + /** + * Decodes a {@code F32} tensor's data. + * + * @param name The tensor's name. Must not be {@code null}. + * @return The tensor's elements in row-major (shape outermost-first) order. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, not a tensor in + * this file, or not declared with dtype {@code F32}. + */ + public float[] readFloat32(String name) { + final TensorInfo info = tensorInfo(name); + if (!"F32".equals(info.dtype())) { + throw new IllegalArgumentException( + "Tensor '" + name + "' has dtype " + info.dtype() + ", not F32"); + } + final long elementCount = info.elementCount(); + final long byteLength = info.dataOffsetEnd() - info.dataOffsetBegin(); + if (byteLength != elementCount * 4L) { + throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount + + " F32 elements but its data range is " + byteLength + " bytes"); + } + final float[] values = new float[(int) elementCount]; + final ByteBuffer buffer = ByteBuffer.wrap(bytes, + (int) (dataStart + info.dataOffsetBegin()), (int) byteLength) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.asFloatBuffer().get(values); + return values; + } + + /** + * Finds the single 2-dimensional {@code F32} tensor in this file, the shape a static + * embedding table's weight matrix takes (vocabulary size by hidden dimension). Deliberately + * strict rather than guessing a name convention: distillation tools do not agree on one, and a + * wrong guess would silently load the wrong tensor. + * + * @return The name of the single 2-D F32 tensor. + * @throws IllegalArgumentException Thrown if the file has zero or more than one 2-D F32 + * tensor; the message lists every candidate so the caller can pick explicitly with + * {@link #readFloat32(String)}. + */ + public String singleMatrixTensorName() { + String found = null; + for (final TensorInfo info : tensorsByName.values()) { + if ("F32".equals(info.dtype()) && info.shape().length == 2) { + if (found != null) { + throw new IllegalArgumentException( + "More than one 2-D F32 tensor in this file; specify the name explicitly. " + + "Candidates: " + tensorsByName.keySet()); + } + found = info.name(); + } + } + if (found == null) { + throw new IllegalArgumentException( + "No 2-D F32 tensor in this file. Available tensors: " + tensorsByName.keySet()); + } + return found; + } + + /** {@return the file's {@code __metadata__} string map, empty when the header has none} */ + public Map metadata() { + return metadata; + } + + /** {@return the total number of tensors declared in this file} */ + public int size() { + return tensorsByName.size(); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java new file mode 100644 index 0000000000..6c05072d7b --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A cursor parser for the JSON header of a safetensors file. Purpose-built for the header's + * fixed, shallow shape (a flat object of tensor name to a {@code dtype}/{@code shape}/ + * {@code data_offsets} record, plus an optional {@code __metadata__} string map), not a + * general-purpose JSON parser: no floating-point numbers, no arbitrary nesting depth, no + * comments. This is the same discipline used by every other data-file cursor parser in the + * project (no regular expressions, fail loud on malformed input). + */ +final class SafetensorsHeaderParser { + + private static final String METADATA_KEY = "__metadata__"; + + private final String text; + private int position; + + private SafetensorsHeaderParser(String text) { + this.text = text; + } + + /** + * Parses a safetensors header. + * + * @param headerJson The header's JSON text, decoded from the file's header bytes. Must not be + * {@code null}. + * @return The parse result: the declared tensors, in header order, and the + * {@code __metadata__} string map (empty when the header has none). + * @throws IllegalArgumentException Thrown if {@code headerJson} is {@code null} or malformed. + */ + static Result parse(String headerJson) { + if (headerJson == null) { + throw new IllegalArgumentException("HeaderJson must not be null"); + } + final SafetensorsHeaderParser parser = new SafetensorsHeaderParser(headerJson); + return parser.parseTop(); + } + + private Result parseTop() { + final List tensors = new ArrayList<>(); + Map metadata = Map.of(); + skipWhitespace(); + expect('{'); + skipWhitespace(); + if (peek() == '}') { + position++; + return new Result(tensors, metadata); + } + while (true) { + skipWhitespace(); + final String key = parseString(); + skipWhitespace(); + expect(':'); + skipWhitespace(); + if (METADATA_KEY.equals(key)) { + metadata = parseStringMap(); + } + else { + tensors.add(parseTensorInfo(key)); + } + skipWhitespace(); + final char next = consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw malformed("Expected ',' or '}' after a header entry, got '" + next + "'"); + } + return new Result(tensors, metadata); + } + + private TensorInfo parseTensorInfo(String name) { + expect('{'); + String dtype = null; + int[] shape = null; + long dataOffsetBegin = -1; + long dataOffsetEnd = -1; + skipWhitespace(); + while (peek() != '}') { + skipWhitespace(); + final String field = parseString(); + skipWhitespace(); + expect(':'); + skipWhitespace(); + switch (field) { + case "dtype" -> dtype = parseString(); + case "shape" -> shape = parseIntArray(); + case "data_offsets" -> { + final long[] offsets = parseLongArray(); + if (offsets.length != 2) { + throw malformed("Tensor '" + name + "' data_offsets must have exactly 2 elements, " + + "got " + offsets.length); + } + dataOffsetBegin = offsets[0]; + dataOffsetEnd = offsets[1]; + } + default -> skipValue(); + } + skipWhitespace(); + final char next = consume(); + if (next == ',') { + skipWhitespace(); + continue; + } + if (next == '}') { + if (dtype == null || shape == null || dataOffsetBegin < 0) { + throw malformed("Tensor '" + name + + "' is missing dtype, shape, or data_offsets"); + } + return new TensorInfo(name, dtype, shape, dataOffsetBegin, dataOffsetEnd); + } + throw malformed("Expected ',' or '}' in tensor '" + name + "', got '" + next + "'"); + } + throw malformed("Tensor '" + name + "' has an empty object; missing dtype, shape, " + + "and data_offsets"); + } + + private Map parseStringMap() { + final Map map = new LinkedHashMap<>(); + expect('{'); + skipWhitespace(); + if (peek() == '}') { + position++; + return map; + } + while (true) { + skipWhitespace(); + final String key = parseString(); + skipWhitespace(); + expect(':'); + skipWhitespace(); + map.put(key, parseString()); + skipWhitespace(); + final char next = consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return map; + } + throw malformed("Expected ',' or '}' in __metadata__, got '" + next + "'"); + } + } + + private int[] parseIntArray() { + final long[] longs = parseLongArray(); + final int[] ints = new int[longs.length]; + for (int i = 0; i < longs.length; i++) { + if (longs[i] < 0 || longs[i] > Integer.MAX_VALUE) { + throw malformed("Shape dimension out of int range: " + longs[i]); + } + ints[i] = (int) longs[i]; + } + return ints; + } + + private long[] parseLongArray() { + expect('['); + skipWhitespace(); + final List values = new ArrayList<>(); + if (peek() == ']') { + position++; + return new long[0]; + } + while (true) { + skipWhitespace(); + values.add(parseLong()); + skipWhitespace(); + final char next = consume(); + if (next == ',') { + continue; + } + if (next == ']') { + break; + } + throw malformed("Expected ',' or ']' in a number array, got '" + next + "'"); + } + final long[] array = new long[values.size()]; + for (int i = 0; i < array.length; i++) { + array[i] = values.get(i); + } + return array; + } + + private long parseLong() { + final int start = position; + if (peek() == '-') { + position++; + } + if (position >= text.length() || !Character.isDigit(text.charAt(position))) { + throw malformed("Expected a non-negative integer"); + } + while (position < text.length() && Character.isDigit(text.charAt(position))) { + position++; + } + try { + return Long.parseLong(text.substring(start, position)); + } + catch (NumberFormatException e) { + throw malformed("Malformed integer: " + text.substring(start, position)); + } + } + + private String parseString() { + expect('"'); + final StringBuilder value = new StringBuilder(); + while (true) { + if (position >= text.length()) { + throw malformed("Unterminated string"); + } + final char c = text.charAt(position++); + if (c == '"') { + return value.toString(); + } + if (c == '\\') { + value.append(parseEscape()); + } + else { + value.append(c); + } + } + } + + private char parseEscape() { + if (position >= text.length()) { + throw malformed("Unterminated escape sequence"); + } + final char escape = text.charAt(position++); + return switch (escape) { + case '"' -> '"'; + case '\\' -> '\\'; + case '/' -> '/'; + case 'b' -> '\b'; + case 'f' -> '\f'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 't' -> '\t'; + case 'u' -> parseUnicodeEscape(); + default -> throw malformed("Unknown escape sequence: \\" + escape); + }; + } + + private char parseUnicodeEscape() { + if (position + 4 > text.length()) { + throw malformed("Truncated \\u escape sequence"); + } + final String hex = text.substring(position, position + 4); + position += 4; + try { + return (char) Integer.parseInt(hex, 16); + } + catch (NumberFormatException e) { + throw malformed("Malformed \\u escape sequence: " + hex); + } + } + + // Skips one JSON value of any type (string, number, array, object, true/false/null); used for + // header fields the reader does not care about (safetensors may add fields over time). + private void skipValue() { + skipWhitespace(); + final char c = peek(); + if (c == '"') { + parseString(); + } + else if (c == '[') { + position++; + skipWhitespace(); + if (peek() != ']') { + while (true) { + skipValue(); + skipWhitespace(); + final char next = consume(); + if (next == ',') { + skipWhitespace(); + continue; + } + if (next == ']') { + return; + } + throw malformed("Expected ',' or ']' while skipping an array, got '" + next + "'"); + } + } + position++; + } + else if (c == '{') { + position++; + skipWhitespace(); + if (peek() != '}') { + while (true) { + skipWhitespace(); + parseString(); + skipWhitespace(); + expect(':'); + skipValue(); + skipWhitespace(); + final char next = consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return; + } + throw malformed("Expected ',' or '}' while skipping an object, got '" + next + "'"); + } + } + position++; + } + else if (c == '-' || Character.isDigit(c)) { + position++; + while (position < text.length() && "0123456789.eE+-".indexOf(text.charAt(position)) >= 0) { + position++; + } + } + else if (text.startsWith("true", position)) { + position += 4; + } + else if (text.startsWith("false", position)) { + position += 5; + } + else if (text.startsWith("null", position)) { + position += 4; + } + else { + throw malformed("Unexpected character while skipping a value: '" + c + "'"); + } + } + + private void skipWhitespace() { + while (position < text.length() && Character.isWhitespace(text.charAt(position))) { + position++; + } + } + + private char peek() { + if (position >= text.length()) { + throw malformed("Unexpected end of header"); + } + return text.charAt(position); + } + + private char consume() { + final char c = peek(); + position++; + return c; + } + + private void expect(char c) { + final char actual = consume(); + if (actual != c) { + throw malformed("Expected '" + c + "', got '" + actual + "'"); + } + } + + private IllegalArgumentException malformed(String message) { + return new IllegalArgumentException( + "Malformed safetensors header at offset " + position + ": " + message); + } + + /** + * The parsed header: the declared tensors, in header order, and the {@code __metadata__} + * string map. + * + * @param tensors The declared tensors, in header order. Never {@code null}. + * @param metadata The {@code __metadata__} string map, empty when the header has none. Never + * {@code null}. + */ + record Result(List tensors, Map metadata) { + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java new file mode 100644 index 0000000000..64c0bb8d89 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +/** + * Header metadata for one tensor in a safetensors file, as declared by the file's own JSON + * header. Carries no data; {@link SafetensorsFile#readFloat32(String)} resolves the bytes. + * + * @param name The tensor's name, the key it was declared under. Never {@code null}. + * @param dtype The declared element type (e.g. {@code "F32"}, {@code "F16"}, + * {@code "I64"}), exactly as written in the header. Never {@code null}. + * @param shape The tensor's dimensions, outermost first. Never {@code null}; empty + * for a scalar. + * @param dataOffsetBegin Start byte offset into the file's data section (relative to the end + * of the header, not the start of the file). + * @param dataOffsetEnd End byte offset (exclusive) into the data section. + */ +public record TensorInfo(String name, String dtype, int[] shape, long dataOffsetBegin, + long dataOffsetEnd) { + + /** + * @return The number of elements the tensor holds, the product of {@link #shape()}. + */ + public long elementCount() { + long count = 1; + for (int dimension : shape) { + count *= dimension; + } + return count; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java new file mode 100644 index 0000000000..d7f55dfa30 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SafetensorsFileTest { + + // Builds a well-formed safetensors file: an 8-byte little-endian header length, the header + // JSON verbatim, then the raw data bytes. The header's data_offsets are expected to already + // be correct for the given data layout; callers construct both together. + private static Path writeFile(Path dir, String name, String headerJson, byte[] data) + throws IOException { + final byte[] headerBytes = headerJson.getBytes(StandardCharsets.UTF_8); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(headerBytes.length).array()); + out.write(headerBytes); + out.write(data); + final Path file = dir.resolve(name); + Files.write(file, out.toByteArray()); + return file; + } + + private static byte[] floatsToLittleEndianBytes(float... values) { + final ByteBuffer buffer = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (float value : values) { + buffer.putFloat(value); + } + return buffer.array(); + } + + @Test + void testRoundTripsAFloat32Matrix(@TempDir Path dir) throws IOException { + final float[] values = {1f, 2f, 3f, 4f, 5f, 6f}; + final byte[] data = floatsToLittleEndianBytes(values); + final String header = "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2,3]," + + "\"data_offsets\":[0," + data.length + "]}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertEquals(1, parsed.size()); + assertEquals(Set.of("weight"), parsed.tensorNames()); + final TensorInfo info = parsed.tensorInfo("weight"); + assertEquals("F32", info.dtype()); + assertArrayEquals(new int[] {2, 3}, info.shape()); + assertEquals(6, info.elementCount()); + assertArrayEquals(values, parsed.readFloat32("weight")); + } + + @Test + void testMultipleTensorsPreserveHeaderOrder(@TempDir Path dir) throws IOException { + final byte[] a = floatsToLittleEndianBytes(1f, 2f); + final byte[] b = floatsToLittleEndianBytes(3f, 4f, 5f); + final String header = "{\"first\":{\"dtype\":\"F32\",\"shape\":[2]," + + "\"data_offsets\":[0," + a.length + "]}," + + "\"second\":{\"dtype\":\"F32\",\"shape\":[3]," + + "\"data_offsets\":[" + a.length + "," + (a.length + b.length) + "]}}"; + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + data.write(a); + data.write(b); + final Path file = writeFile(dir, "model.safetensors", header, data.toByteArray()); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertEquals(java.util.List.of("first", "second"), java.util.List.copyOf(parsed.tensorNames())); + assertArrayEquals(new float[] {1f, 2f}, parsed.readFloat32("first")); + assertArrayEquals(new float[] {3f, 4f, 5f}, parsed.readFloat32("second")); + } + + @Test + void testMetadataMapIsParsed(@TempDir Path dir) throws IOException { + final byte[] data = floatsToLittleEndianBytes(1f); + final String header = "{\"__metadata__\":{\"format\":\"pt\",\"note\":\"line\\nbreak\"}," + + "\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0," + data.length + "]}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertEquals("pt", parsed.metadata().get("format")); + assertEquals("line\nbreak", parsed.metadata().get("note")); + assertEquals(1, parsed.size()); + } + + @Test + void testUnknownHeaderFieldsAreSkipped(@TempDir Path dir) throws IOException { + final byte[] data = floatsToLittleEndianBytes(1f, 2f); + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2]," + + "\"data_offsets\":[0," + data.length + "],\"future_field\":{\"nested\":[1,2,3]}}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertArrayEquals(new float[] {1f, 2f}, parsed.readFloat32("w")); + } + + @Test + void testSingleMatrixTensorNameFindsTheOnly2DFloat32Tensor(@TempDir Path dir) throws IOException { + final byte[] scalar = floatsToLittleEndianBytes(9f); + final byte[] matrix = floatsToLittleEndianBytes(1f, 2f, 3f, 4f); + final String header = "{\"bias\":{\"dtype\":\"F32\",\"shape\":[1]," + + "\"data_offsets\":[0," + scalar.length + "]}," + + "\"embeddings\":{\"dtype\":\"F32\",\"shape\":[2,2]," + + "\"data_offsets\":[" + scalar.length + "," + (scalar.length + matrix.length) + "]}}"; + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + data.write(scalar); + data.write(matrix); + final Path file = writeFile(dir, "model.safetensors", header, data.toByteArray()); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertEquals("embeddings", parsed.singleMatrixTensorName()); + } + + @Test + void testSingleMatrixTensorNameRejectsAmbiguity(@TempDir Path dir) throws IOException { + final byte[] a = floatsToLittleEndianBytes(1f, 2f, 3f, 4f); + final byte[] b = floatsToLittleEndianBytes(5f, 6f, 7f, 8f); + final String header = "{\"a\":{\"dtype\":\"F32\",\"shape\":[2,2]," + + "\"data_offsets\":[0," + a.length + "]}," + + "\"b\":{\"dtype\":\"F32\",\"shape\":[2,2]," + + "\"data_offsets\":[" + a.length + "," + (a.length + b.length) + "]}}"; + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + data.write(a); + data.write(b); + final Path file = writeFile(dir, "model.safetensors", header, data.toByteArray()); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertThrows(IllegalArgumentException.class, parsed::singleMatrixTensorName); + } + + @Test + void testSingleMatrixTensorNameRejectsNoCandidate(@TempDir Path dir) throws IOException { + final byte[] data = floatsToLittleEndianBytes(1f); + final String header = "{\"bias\":{\"dtype\":\"F32\",\"shape\":[1]," + + "\"data_offsets\":[0," + data.length + "]}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertThrows(IllegalArgumentException.class, parsed::singleMatrixTensorName); + } + + @Test + void testReadFloat32RejectsWrongDtype(@TempDir Path dir) throws IOException { + final byte[] data = new byte[] {1, 2}; + final String header = "{\"ids\":{\"dtype\":\"I64\",\"shape\":[1]," + + "\"data_offsets\":[0,2]}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("ids")); + assertTrue(e.getMessage().contains("I64")); + } + + @Test + void testTensorInfoRejectsUnknownName(@TempDir Path dir) throws IOException { + final byte[] data = floatsToLittleEndianBytes(1f); + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + assertThrows(IllegalArgumentException.class, () -> parsed.tensorInfo("missing")); + } + + @Test + void testRejectsNullAndMissingFile(@TempDir Path dir) { + assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(null)); + assertThrows(IllegalArgumentException.class, + () -> SafetensorsFile.read(dir.resolve("absent.safetensors"))); + } + + @Test + void testRejectsFileShorterThanTheLengthPrefix(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("truncated.safetensors"); + Files.write(file, new byte[] {1, 2, 3}); + + assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + } + + @Test + void testRejectsHeaderLengthLargerThanTheFile(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("bad-length.safetensors"); + final byte[] prefix = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(1000L).array(); + Files.write(file, prefix); + + assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + } + + @Test + void testRejectsDuplicateTensorName(@TempDir Path dir) throws IOException { + // The same key twice is syntactically valid JSON (just semantically ambiguous), so the + // header parser itself does not reject it; SafetensorsFile's post-parse check does. + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}," + + "\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; + final Path file = writeFile(dir, "model.safetensors", header, new byte[] {1, 2, 3, 4}); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + assertTrue(e.getMessage().contains("more than once")); + } + + @Test + void testRejectsTensorMissingRequiredField(@TempDir Path dir) throws IOException { + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]}}"; + final Path file = writeFile(dir, "model.safetensors", header, new byte[0]); + + assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + } + + @Test + void testRejectsDataOffsetsOutOfRange(@TempDir Path dir) throws IOException { + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,999]}}"; + final Path file = writeFile(dir, "model.safetensors", header, new byte[] {1, 2, 3, 4}); + + assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + } + + @Test + void testRejectsUnterminatedString(@TempDir Path dir) throws IOException { + final String header = "{\"w\":{\"dtype\":\"F32"; + final Path file = writeFile(dir, "model.safetensors", header, new byte[0]); + + assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + } +} diff --git a/opennlp-extensions/pom.xml b/opennlp-extensions/pom.xml index 2b0e87273d..4fed4c405e 100644 --- a/opennlp-extensions/pom.xml +++ b/opennlp-extensions/pom.xml @@ -38,6 +38,7 @@ + opennlp-embeddings opennlp-morfologik opennlp-spellcheck opennlp-subword From dc3fa779652d09d3538be7182980bb33c0270460 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 12:04:40 -0400 Subject: [PATCH 25/82] Add StaticEmbeddingModel: tokenize, gather, weight, mean-pool, normalize WordPieceVocabulary reads a BERT-style vocab.txt (line number is the token's row id, the format minishlab/potion-base-8M and the wider BGE/BERT family ship). StaticEmbeddingModel wires it to the existing BertTokenizer/WordpieceTokenizer (reused as-is, no new tokenizer code) and the safetensors reader from the previous commit, and implements the pooling formula. The formula is verified against MinishLab's Rust reference implementation (model2vec-rs), not assumed: [CLS]/[SEP] are stripped before pooling since this is table lookup, not transformer input (the tokenizer always adds them, so this class trims the first/last token rather than needing a second tokenizer mode); unknown tokens are dropped from both the sum and the denominator; each pooled token's vector is multiplied by an optional per-token weight from a second "weights" tensor when the safetensors file has one; the sum is divided by the plain pooled-token count, not the sum of weights, which is the exact detail source-verification caught (the two give different results whenever a weight isn't 1.0, and guessing wrong would have silently produced vectors that don't match the reference Python/Rust output). Normalization uses an epsilon floor so a token-less input yields a zero vector instead of a division by zero. Tests hand-compute the expected pooled vectors for a small synthetic vocabulary and safetensors fixture, including a dedicated test that distinguishes the weighted-sum/token-count-denominator behavior from the (wrong) weighted-sum/sum-of-weights alternative. --- .../embeddings/StaticEmbeddingModel.java | 192 ++++++++++++++ .../embeddings/WordPieceVocabulary.java | 110 ++++++++ .../embeddings/StaticEmbeddingModelTest.java | 241 ++++++++++++++++++ 3 files changed, 543 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java new file mode 100644 index 0000000000..cb238d9241 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.nio.file.Path; +import java.util.OptionalInt; + +import opennlp.tools.tokenize.BertTokenizer; +import opennlp.tools.tokenize.WordpieceTokenizer; + +/** + * A static (non-contextual) sentence embedding model: a per-token vector table plus WordPiece + * tokenization, the pure-JVM word2vec/GloVe successor described in the design doc this module + * implements. Distilled tables in this shape (Model2Vec and compatible releases) carry a modern + * sentence-transformer's semantics in a flat lookup table, so embedding a sentence is tokenize, + * gather, (optionally) weight, mean-pool, and (optionally) normalize: no model forward pass, no + * GPU, no native runtime. + * + *

The pooling formula matches the reference Model2Vec implementations exactly (verified + * against MinishLab's Rust {@code model2vec-rs}, not assumed): {@code [CLS]}/{@code [SEP]} are + * never added to the pool (this class tokenizes for lookup, not for a transformer), unknown + * tokens are dropped rather than contributing a meaningless vector, each remaining token's + * vector is multiplied by its optional per-token weight, the sum is divided by the plain count + * of pooled tokens (not the sum of weights), and if the model calls for normalization the + * pooled vector is L2-normalized with an epsilon floor so a token-less input yields a zero + * vector rather than a division by zero.

+ * + *

Instances are immutable and safe for concurrent {@link #embed(String)} calls after + * construction.

+ */ +public final class StaticEmbeddingModel { + + private static final float NORMALIZE_EPSILON = 1e-12f; + private static final String WEIGHTS_TENSOR_NAME = "weights"; + + private final float[] embeddings; + private final float[] weights; + private final int dimension; + private final WordPieceVocabulary vocabulary; + private final BertTokenizer tokenizer; + private final boolean normalize; + private final String unknownToken; + + private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, + WordPieceVocabulary vocabulary, BertTokenizer tokenizer, + boolean normalize, String unknownToken) { + this.embeddings = embeddings; + this.weights = weights; + this.dimension = dimension; + this.vocabulary = vocabulary; + this.tokenizer = tokenizer; + this.normalize = normalize; + this.unknownToken = unknownToken; + } + + /** + * Loads a static embedding model from a BERT-style {@code vocab.txt} and a safetensors weight + * file, the file pair a Model2Vec-family distillation publishes. No model is bundled with this + * module: the caller points at files they downloaded (see the module's design doc for the + * license posture). + * + * @param vocabularyFile The {@code vocab.txt} file: one token per line, line number is the + * token's row id. Must not be {@code null} and must exist. + * @param safetensorsFile The {@code model.safetensors} file. Must not be {@code null} and + * must exist, and must contain exactly one 2-D {@code F32} tensor + * (the embedding matrix) whose row count matches the vocabulary size. + * An optional 1-D {@code F32} tensor named {@code "weights"}, one + * scalar per vocabulary row, is used as a per-token pooling weight + * when present. + * @param lowerCase Whether the tokenizer should lower-case and strip accents, matching + * the base model's tokenizer configuration ({@code true} for the + * uncased BGE/BERT family this module targets). + * @param normalize Whether {@link #embed(String)} L2-normalizes its result, matching + * the source model's {@code config.json} {@code normalize} field. + * @return The loaded model. + * @throws IllegalArgumentException Thrown if an argument is {@code null}, a file is missing + * or malformed, or the vocabulary size and the embedding matrix's row count disagree. + */ + public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFile, + boolean lowerCase, boolean normalize) { + if (vocabularyFile == null) { + throw new IllegalArgumentException("VocabularyFile must not be null"); + } + if (safetensorsFile == null) { + throw new IllegalArgumentException("SafetensorsFile must not be null"); + } + final WordPieceVocabulary vocabulary = WordPieceVocabulary.read(vocabularyFile); + final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); + + final String matrixName = tensors.singleMatrixTensorName(); + final TensorInfo matrixInfo = tensors.tensorInfo(matrixName); + if (matrixInfo.shape()[0] != vocabulary.size()) { + throw new IllegalArgumentException("Vocabulary " + vocabularyFile + " has " + + vocabulary.size() + " tokens but embedding matrix '" + matrixName + "' in " + + safetensorsFile + " has " + matrixInfo.shape()[0] + " rows; these files do not " + + "belong to the same model"); + } + final int dimension = matrixInfo.shape()[1]; + final float[] embeddings = tensors.readFloat32(matrixName); + + float[] weights = null; + if (tensors.tensorNames().contains(WEIGHTS_TENSOR_NAME)) { + weights = tensors.readFloat32(WEIGHTS_TENSOR_NAME); + if (weights.length != vocabulary.size()) { + throw new IllegalArgumentException("Tensor '" + WEIGHTS_TENSOR_NAME + "' in " + + safetensorsFile + " has " + weights.length + " elements but the vocabulary has " + + vocabulary.size() + " tokens"); + } + } + + final BertTokenizer tokenizer = new BertTokenizer(vocabulary.tokens(), lowerCase); + return new StaticEmbeddingModel(embeddings, weights, dimension, vocabulary, tokenizer, + normalize, WordpieceTokenizer.BERT_UNK_TOKEN); + } + + /** + * Embeds a piece of text. + * + * @param text The text to embed. Must not be {@code null}. + * @return The pooled embedding vector, of length {@link #dimension()}. A text with no + * in-vocabulary tokens yields a zero vector. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + public float[] embed(String text) { + if (text == null) { + throw new IllegalArgumentException("Text must not be null"); + } + // The tokenizer always wraps its output in [CLS] ... [SEP]; neither belongs in the pool + // (this is table lookup, not transformer input), so the first and last tokens are skipped. + final String[] tokens = tokenizer.tokenize(text); + final float[] sum = new float[dimension]; + int pooledCount = 0; + for (int i = 1; i < tokens.length - 1; i++) { + final String token = tokens[i]; + if (unknownToken.equals(token)) { + continue; + } + final OptionalInt id = vocabulary.id(token); + if (id.isEmpty()) { + throw new IllegalStateException("Tokenizer produced token '" + token + + "' that is not in its own vocabulary; this indicates a tokenizer/vocabulary " + + "construction bug, not an input problem"); + } + final int row = id.getAsInt(); + final float weight = weights == null ? 1f : weights[row]; + final int base = row * dimension; + for (int d = 0; d < dimension; d++) { + sum[d] += embeddings[base + d] * weight; + } + pooledCount++; + } + final int denominator = Math.max(pooledCount, 1); + for (int d = 0; d < dimension; d++) { + sum[d] /= denominator; + } + if (normalize) { + double sumOfSquares = 0; + for (final float value : sum) { + sumOfSquares += (double) value * value; + } + final float norm = (float) Math.max(Math.sqrt(sumOfSquares), NORMALIZE_EPSILON); + for (int d = 0; d < dimension; d++) { + sum[d] /= norm; + } + } + return sum; + } + + /** {@return the dimension of every vector this model produces} */ + public int dimension() { + return dimension; + } + + /** {@return the number of tokens in this model's vocabulary} */ + public int vocabularySize() { + return vocabulary.size(); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java new file mode 100644 index 0000000000..b26ba9624e --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalInt; +import java.util.Set; + +/** + * A BERT-style {@code vocab.txt} vocabulary: one token per line, the line number (0-based) is + * the token's id. This is the same file format {@code bert-base-uncased} and the BGE family of + * models ship (the tokenizer {@code minishlab/potion-base-8M} was distilled from), and it is the + * row index into a static-embedding table's weight matrix: row {@code id} is that token's + * vector. + * + *

Immutable and safe for concurrent reads after construction.

+ */ +final class WordPieceVocabulary { + + private final Map idByToken; + + private WordPieceVocabulary(Map idByToken) { + this.idByToken = idByToken; + } + + /** + * Reads a {@code vocab.txt} file. + * + * @param file The vocabulary file. Must not be {@code null} and must exist. + * @return The parsed vocabulary. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null}, missing, or + * contains a duplicate token. + * @throws UncheckedIOException Thrown if reading the file fails. + */ + static WordPieceVocabulary read(Path file) { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); + } + final List lines; + try { + lines = Files.readAllLines(file); + } + catch (IOException e) { + throw new UncheckedIOException("Unable to read vocabulary file " + file, e); + } + return fromLines(lines, file.toString()); + } + + // Package-private so tests can build a vocabulary from in-memory lines without a temp file. + static WordPieceVocabulary fromLines(List lines, String sourceName) { + final Map idByToken = new LinkedHashMap<>(lines.size() * 2); + for (int id = 0; id < lines.size(); id++) { + final String token = lines.get(id); + if (idByToken.putIfAbsent(token, id) != null) { + throw new IllegalArgumentException( + "Vocabulary " + sourceName + " declares token '" + token + + "' more than once, at lines " + idByToken.get(token) + " and " + id); + } + } + return new WordPieceVocabulary(Collections.unmodifiableMap(idByToken)); + } + + /** {@return every token in this vocabulary, suitable for a WordpieceTokenizer} */ + Set tokens() { + return idByToken.keySet(); + } + + /** + * Looks up a token's row id. + * + * @param token The token to look up. Must not be {@code null}. + * @return The token's id, or empty when the token is not in this vocabulary. + */ + OptionalInt id(String token) { + if (token == null) { + throw new IllegalArgumentException("Token must not be null"); + } + final Integer id = idByToken.get(token); + return id == null ? OptionalInt.empty() : OptionalInt.of(id); + } + + /** {@return the number of tokens in this vocabulary} */ + int size() { + return idByToken.size(); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java new file mode 100644 index 0000000000..1424472843 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StaticEmbeddingModelTest { + + // Fixture vocabulary: [CLS]=0, [SEP]=1, [UNK]=2, hello=3, world=4, cat=5. + private static final List VOCAB_TOKENS = + List.of("[CLS]", "[SEP]", "[UNK]", "hello", "world", "cat"); + private static final int DIMENSION = 3; + + // Row i is [i, i*10, i*100], so hand-computed expected pooled vectors are easy to verify. + private static final float[][] ROWS = { + {0f, 0f, 0f}, // [CLS] + {1f, 10f, 100f}, // [SEP] + {2f, 20f, 200f}, // [UNK] + {3f, 30f, 300f}, // hello + {4f, 40f, 400f}, // world + {5f, 50f, 500f}, // cat + }; + + private static Path writeVocab(Path dir) throws IOException { + final Path file = dir.resolve("vocab.txt"); + Files.write(file, VOCAB_TOKENS); + return file; + } + + private static Path writeSafetensors(Path dir, boolean withWeights) throws IOException { + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + final ByteBuffer embeddingBuffer = + ByteBuffer.allocate(ROWS.length * DIMENSION * 4).order(ByteOrder.LITTLE_ENDIAN); + for (final float[] row : ROWS) { + for (final float value : row) { + embeddingBuffer.putFloat(value); + } + } + final byte[] embeddingBytes = embeddingBuffer.array(); + data.write(embeddingBytes); + + String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + ROWS.length + "," + + DIMENSION + "],\"data_offsets\":[0," + embeddingBytes.length + "]}"; + if (withWeights) { + // Weight per row: [1, 1, 1, 2, 1, 1] so "hello" (row 3) counts double in the sum but not + // in the pooling denominator, which is the exact behavior being pinned. + final float[] weightValues = {1f, 1f, 1f, 2f, 1f, 1f}; + final ByteBuffer weightBuffer = + ByteBuffer.allocate(weightValues.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (final float value : weightValues) { + weightBuffer.putFloat(value); + } + final byte[] weightBytes = weightBuffer.array(); + final int start = embeddingBytes.length; + data.write(weightBytes); + header += ",\"weights\":{\"dtype\":\"F32\",\"shape\":[" + weightValues.length + + "],\"data_offsets\":[" + start + "," + (start + weightBytes.length) + "]}"; + } + header += "}"; + + final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(headerBytes.length).array()); + out.write(headerBytes); + out.write(data.toByteArray()); + final Path file = dir.resolve("model.safetensors"); + Files.write(file, out.toByteArray()); + return file; + } + + @Test + void testEmbedMeanPoolsWithoutWeights(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + + final float[] result = model.embed("hello world"); + + // (hello + world) / 2 = ([3,30,300] + [4,40,400]) / 2 = [3.5, 35, 350] + assertArrayEquals(new float[] {3.5f, 35f, 350f}, result, 1e-5f); + } + + @Test + void testEmbedAppliesPerTokenWeightsButDividesByTokenCount(@TempDir Path dir) + throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, true), true, false); + + final float[] result = model.embed("hello world"); + + // hello has weight 2: (2*[3,30,300] + 1*[4,40,400]) / 2 (denominator is token COUNT, not + // the sum of weights) = ([6,60,600] + [4,40,400]) / 2 = [5, 50, 500] + assertArrayEquals(new float[] {5f, 50f, 500f}, result, 1e-5f); + } + + @Test + void testEmbedNormalizesToUnitLength(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, true); + + final float[] result = model.embed("cat"); + + double normSquared = 0; + for (final float v : result) { + normSquared += (double) v * v; + } + assertEquals(1.0, Math.sqrt(normSquared), 1e-5); + // Direction preserved: cat's raw vector is [5, 50, 500], i.e. a positive multiple of + // [1, 10, 100]; the normalized result must be that same direction. + assertTrue(result[1] / result[0] > 9.9f && result[1] / result[0] < 10.1f); + } + + @Test + void testEmbedSkipsUnknownTokens(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + + // "xyzzy" cannot be represented by any vocabulary piece, so it becomes [UNK] and must be + // excluded from both the sum and the pooling denominator, leaving just "cat". + final float[] result = model.embed("cat xyzzy"); + + assertArrayEquals(new float[] {5f, 50f, 500f}, result, 1e-5f); + } + + @Test + void testEmbedOfTextWithNoInVocabularyTokensIsZeroVector(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + + assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed("xyzzy"), 1e-5f); + } + + @Test + void testEmbedOfEmptyTextIsZeroVectorNotAnError(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, true); + + assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed(""), 1e-5f); + } + + @Test + void testDimensionAndVocabularySizeAccessors(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + + assertEquals(DIMENSION, model.dimension()); + assertEquals(VOCAB_TOKENS.size(), model.vocabularySize()); + } + + @Test + void testEmbedRejectsNullText(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + + assertThrows(IllegalArgumentException.class, () -> model.embed(null)); + } + + @Test + void testLoadRejectsNullArguments(@TempDir Path dir) throws IOException { + final Path vocab = writeVocab(dir); + final Path tensors = writeSafetensors(dir, false); + + assertThrows(IllegalArgumentException.class, + () -> StaticEmbeddingModel.load(null, tensors, true, false)); + assertThrows(IllegalArgumentException.class, + () -> StaticEmbeddingModel.load(vocab, null, true, false)); + } + + @Test + void testLoadRejectsVocabularySizeMismatch(@TempDir Path dir) throws IOException { + final Path shortVocab = dir.resolve("short-vocab.txt"); + Files.write(shortVocab, List.of("[CLS]", "[SEP]", "[UNK]")); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> StaticEmbeddingModel.load(shortVocab, writeSafetensors(dir, false), true, false)); + assertTrue(e.getMessage().contains("rows")); + } + + @Test + void testLoadRejectsWeightsSizeMismatch(@TempDir Path dir) throws IOException { + // A weights tensor sized for a different (smaller) vocabulary than the embedding matrix. + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + final ByteBuffer embeddingBuffer = + ByteBuffer.allocate(ROWS.length * DIMENSION * 4).order(ByteOrder.LITTLE_ENDIAN); + for (final float[] row : ROWS) { + for (final float value : row) { + embeddingBuffer.putFloat(value); + } + } + final byte[] embeddingBytes = embeddingBuffer.array(); + data.write(embeddingBytes); + final byte[] weightBytes = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN) + .putFloat(1f).array(); + data.write(weightBytes); + final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + ROWS.length + "," + + DIMENSION + "],\"data_offsets\":[0," + embeddingBytes.length + "]}," + + "\"weights\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[" + + embeddingBytes.length + "," + (embeddingBytes.length + weightBytes.length) + "]}}"; + final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(headerBytes.length).array()); + out.write(headerBytes); + out.write(data.toByteArray()); + final Path file = dir.resolve("mismatched.safetensors"); + Files.write(file, out.toByteArray()); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> StaticEmbeddingModel.load(writeVocab(dir), file, true, false)); + assertTrue(e.getMessage().contains("weights")); + } +} From cffed1fcd41aad60011f9ecfd30856802b7efaca Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 12:09:31 -0400 Subject: [PATCH 26/82] Add word similarity and analogy convenience API similarity(text1, text2): cosine similarity between two pooled embeddings. mostSimilar(text, topK): nearest vocabulary tokens to a pooled query vector, brute-force over the vocabulary (fine at the tens-of- thousands-of-rows scale this module targets; an ANN index is a documented, deferred follow-up, not v1 scope). Excludes the special tokens only; a single-word query's own vocabulary row is, correctly, its own top match, unlike gensim's convention of excluding the query word, which does not generalize to multi-word text queries anyway. analogy(a, b, c, topK): the classic word2vec vector arithmetic (embed(b) - embed(a) + embed(c)), additionally excluding a, b, and c themselves from the results, which is load-bearing here (not just convention) since all three are trivially close to the constructed target vector. Tests use a small fixture with genuinely non-collinear vectors (the pooling-math fixture in StaticEmbeddingModelTest is deliberately collinear, which is ideal for hand-computing weighted averages but would make every pairwise similarity a trivial 1.0), built so the analogy has an exact answer: king - man + woman == queen. --- .../java/opennlp/embeddings/Neighbor.java | 27 +++ .../embeddings/StaticEmbeddingModel.java | 139 ++++++++++++ .../embeddings/WordPieceVocabulary.java | 16 +- .../StaticEmbeddingModelSimilarityTest.java | 213 ++++++++++++++++++ 4 files changed, 393 insertions(+), 2 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java new file mode 100644 index 0000000000..68d6f3d58e --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +/** + * One vocabulary token found near a query vector by {@link StaticEmbeddingModel#mostSimilar} + * or {@link StaticEmbeddingModel#analogy}, most similar first. + * + * @param token The vocabulary token (a single WordPiece, not necessarily a whole word). + * @param similarity Cosine similarity to the query vector, in {@code [-1, 1]}. + */ +public record Neighbor(String token, double similarity) { +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index cb238d9241..51d4d8c45e 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -17,7 +17,11 @@ package opennlp.embeddings; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; import java.util.OptionalInt; +import java.util.Set; import opennlp.tools.tokenize.BertTokenizer; import opennlp.tools.tokenize.WordpieceTokenizer; @@ -46,6 +50,9 @@ public final class StaticEmbeddingModel { private static final float NORMALIZE_EPSILON = 1e-12f; private static final String WEIGHTS_TENSOR_NAME = "weights"; + // Never meaningful as a "similar word" result. + private static final Set SPECIAL_TOKENS = Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, + WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN); private final float[] embeddings; private final float[] weights; @@ -189,4 +196,136 @@ public int dimension() { public int vocabularySize() { return vocabulary.size(); } + + /** + * Cosine similarity between two pieces of text's pooled embeddings, the classic word2vec-era + * convenience this module exists to modernize. + * + * @param text1 The first text. Must not be {@code null}. + * @param text2 The second text. Must not be {@code null}. + * @return The cosine similarity, in {@code [-1, 1]}; {@code 0} when either text has no + * in-vocabulary tokens (an undefined direction, not an error). + * @throws IllegalArgumentException Thrown if {@code text1} or {@code text2} is {@code null}. + */ + public double similarity(String text1, String text2) { + if (text1 == null) { + throw new IllegalArgumentException("Text1 must not be null"); + } + if (text2 == null) { + throw new IllegalArgumentException("Text2 must not be null"); + } + return cosineSimilarity(embed(text1), embed(text2)); + } + + /** + * Finds the vocabulary tokens whose vectors are nearest a piece of text's pooled embedding, + * most similar first. A brute-force scan over the whole vocabulary; fine for the vocabulary + * sizes this module targets (tens of thousands of rows), not an approximate-nearest-neighbor + * index (a documented follow-up, not v1 scope). + * + * @param text The query text. Must not be {@code null}. + * @param topK The maximum number of results. Must be at least 1. + * @return Up to {@code topK} neighbors, most similar first, excluding the special tokens + * ({@code [CLS]}, {@code [SEP]}, {@code [UNK]}); empty when {@code text} has no + * in-vocabulary tokens. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null} or {@code topK} is + * less than 1. + */ + public List mostSimilar(String text, int topK) { + if (text == null) { + throw new IllegalArgumentException("Text must not be null"); + } + requirePositive(topK); + return nearestNeighbors(embed(text), topK, Set.of()); + } + + /** + * The classic word2vec analogy: {@code b} is to {@code a} as the results are to {@code c} + * (computed as {@code embed(b) - embed(a) + embed(c)}), for example {@code analogy("man", + * "king", "woman", 1)} for "man is to king as woman is to ?". + * + * @param a The first term. Must not be {@code null}. + * @param b The second term. Must not be {@code null}. + * @param c The third term. Must not be {@code null}. + * @param topK The maximum number of results. Must be at least 1. + * @return Up to {@code topK} neighbors, most similar first, excluding the special tokens and + * any vocabulary token that exactly matches {@code a}, {@code b}, or {@code c}. + * @throws IllegalArgumentException Thrown if {@code a}, {@code b}, or {@code c} is + * {@code null}, or {@code topK} is less than 1. + */ + public List analogy(String a, String b, String c, int topK) { + if (a == null) { + throw new IllegalArgumentException("A must not be null"); + } + if (b == null) { + throw new IllegalArgumentException("B must not be null"); + } + if (c == null) { + throw new IllegalArgumentException("C must not be null"); + } + requirePositive(topK); + final float[] va = embed(a); + final float[] vb = embed(b); + final float[] vc = embed(c); + final float[] target = new float[dimension]; + for (int d = 0; d < dimension; d++) { + target[d] = vb[d] - va[d] + vc[d]; + } + return nearestNeighbors(target, topK, Set.of(a, b, c)); + } + + private static void requirePositive(int topK) { + if (topK < 1) { + throw new IllegalArgumentException("TopK must be at least 1, got " + topK); + } + } + + private List nearestNeighbors(float[] query, int topK, Set exclude) { + final double queryNorm = norm(query); + if (queryNorm < NORMALIZE_EPSILON) { + return List.of(); + } + final List candidates = new ArrayList<>(vocabulary.size()); + for (int row = 0; row < vocabulary.size(); row++) { + final String token = vocabulary.token(row); + if (SPECIAL_TOKENS.contains(token) || exclude.contains(token)) { + continue; + } + final int base = row * dimension; + double dot = 0; + double rowNormSquared = 0; + for (int d = 0; d < dimension; d++) { + final float value = embeddings[base + d]; + dot += query[d] * value; + rowNormSquared += (double) value * value; + } + final double rowNorm = Math.sqrt(rowNormSquared); + final double similarity = rowNorm < NORMALIZE_EPSILON ? 0.0 : dot / (queryNorm * rowNorm); + candidates.add(new Neighbor(token, similarity)); + } + candidates.sort(Comparator.comparingDouble(Neighbor::similarity).reversed()); + return topK >= candidates.size() ? List.copyOf(candidates) + : List.copyOf(candidates.subList(0, topK)); + } + + private static double cosineSimilarity(float[] a, float[] b) { + double dot = 0; + double normASquared = 0; + double normBSquared = 0; + for (int d = 0; d < a.length; d++) { + dot += (double) a[d] * b[d]; + normASquared += (double) a[d] * a[d]; + normBSquared += (double) b[d] * b[d]; + } + final double denominator = Math.sqrt(normASquared) * Math.sqrt(normBSquared); + return denominator < NORMALIZE_EPSILON ? 0.0 : dot / denominator; + } + + private static double norm(float[] vector) { + double sumOfSquares = 0; + for (final float value : vector) { + sumOfSquares += (double) value * value; + } + return Math.sqrt(sumOfSquares); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java index b26ba9624e..10581b70ac 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java @@ -39,9 +39,11 @@ final class WordPieceVocabulary { private final Map idByToken; + private final List tokenById; - private WordPieceVocabulary(Map idByToken) { + private WordPieceVocabulary(Map idByToken, List tokenById) { this.idByToken = idByToken; + this.tokenById = tokenById; } /** @@ -81,7 +83,7 @@ static WordPieceVocabulary fromLines(List lines, String sourceName) { + "' more than once, at lines " + idByToken.get(token) + " and " + id); } } - return new WordPieceVocabulary(Collections.unmodifiableMap(idByToken)); + return new WordPieceVocabulary(Collections.unmodifiableMap(idByToken), List.copyOf(lines)); } /** {@return every token in this vocabulary, suitable for a WordpieceTokenizer} */ @@ -107,4 +109,14 @@ OptionalInt id(String token) { int size() { return idByToken.size(); } + + /** + * Looks up the token at a row id. + * + * @param id The row id. Must be within {@code [0, size())}. + * @return The token at that id. + */ + String token(int id) { + return tokenById.get(id); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java new file mode 100644 index 0000000000..103788c679 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises {@link StaticEmbeddingModel#similarity}, {@link StaticEmbeddingModel#mostSimilar}, + * and {@link StaticEmbeddingModel#analogy} against a small fixture whose vectors point in + * genuinely different directions (unlike {@link StaticEmbeddingModelTest}'s collinear rows, + * which are ideal for pooling-math assertions but would make every pairwise cosine similarity + * trivially 1.0). The fixture is built so the classic word2vec analogy has an exact answer: + * {@code king - man + woman == queen}. + */ +class StaticEmbeddingModelSimilarityTest { + + private static final List VOCAB_TOKENS = + List.of("[CLS]", "[SEP]", "[UNK]", "king", "queen", "man", "woman", "apple"); + private static final int DIMENSION = 2; + + // king - man + woman = [3,3] - [2,1] + [1,2] = [2,4] = queen, exactly. + private static final float[][] ROWS = { + {0f, 0f}, // [CLS] + {0f, 0f}, // [SEP] + {0f, 0f}, // [UNK] + {3f, 3f}, // king + {2f, 4f}, // queen + {2f, 1f}, // man + {1f, 2f}, // woman + {-3f, -1f}, // apple: unrelated, opposite-ish direction + }; + + private static Path writeVocab(Path dir) throws IOException { + final Path file = dir.resolve("vocab.txt"); + Files.write(file, VOCAB_TOKENS); + return file; + } + + private static Path writeSafetensors(Path dir) throws IOException { + final ByteBuffer buffer = + ByteBuffer.allocate(ROWS.length * DIMENSION * 4).order(ByteOrder.LITTLE_ENDIAN); + for (final float[] row : ROWS) { + for (final float value : row) { + buffer.putFloat(value); + } + } + final byte[] data = buffer.array(); + final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + ROWS.length + "," + + DIMENSION + "],\"data_offsets\":[0," + data.length + "]}}"; + final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(headerBytes.length).array()); + out.write(headerBytes); + out.write(data); + final Path file = dir.resolve("model.safetensors"); + Files.write(file, out.toByteArray()); + return file; + } + + private static StaticEmbeddingModel load(Path dir) throws IOException { + return StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir), true, false); + } + + @Test + void testSimilarityOfIdenticalTextIsOne(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertEquals(1.0, model.similarity("king", "king"), 1e-5); + } + + @Test + void testSimilarityIsSymmetric(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertEquals(model.similarity("king", "queen"), model.similarity("queen", "king"), 1e-9); + } + + @Test + void testSimilarityOfUnrelatedTermsIsLow(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertTrue(model.similarity("king", "apple") < model.similarity("king", "queen")); + } + + @Test + void testSimilarityOfOutOfVocabularyTextIsZero(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertEquals(0.0, model.similarity("xyzzy", "king"), 1e-9); + } + + @Test + void testMostSimilarFindsSelfAsTopMatch(@TempDir Path dir) throws IOException { + // Documented, deliberate behavior: unlike gensim's convention of excluding the query word, + // mostSimilar only excludes special tokens, so a single-word query's own vocabulary row is + // (correctly) its own nearest neighbor. + final StaticEmbeddingModel model = load(dir); + + final List result = model.mostSimilar("king", 1); + + assertEquals(1, result.size()); + assertEquals("king", result.get(0).token()); + assertEquals(1.0, result.get(0).similarity(), 1e-5); + } + + @Test + void testMostSimilarExcludesSpecialTokensAndOrdersByDescendingSimilarity(@TempDir Path dir) + throws IOException { + final StaticEmbeddingModel model = load(dir); + + final List result = model.mostSimilar("king", 5); + + assertEquals(5, result.size()); + for (final Neighbor neighbor : result) { + assertFalse(List.of("[CLS]", "[SEP]", "[UNK]").contains(neighbor.token())); + } + // Descending order. + for (int i = 1; i < result.size(); i++) { + assertTrue(result.get(i - 1).similarity() >= result.get(i).similarity()); + } + // apple is the clear outlier (opposite-ish direction) and must rank last. + assertEquals("apple", result.get(result.size() - 1).token()); + } + + @Test + void testMostSimilarOfZeroVectorQueryReturnsEmptyList(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertEquals(List.of(), model.mostSimilar("xyzzy", 3)); + } + + @Test + void testAnalogyFindsTheExactTarget(@TempDir Path dir) throws IOException { + // man is to king as woman is to ? Expected: queen (king - man + woman == queen exactly). + final StaticEmbeddingModel model = load(dir); + + final List result = model.analogy("man", "king", "woman", 1); + + assertEquals(1, result.size()); + assertEquals("queen", result.get(0).token()); + assertEquals(1.0, result.get(0).similarity(), 1e-5); + } + + @Test + void testAnalogyExcludesItsOwnInputTerms(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + // Only "queen" and "apple" remain eligible once man/king/woman and the special tokens are + // excluded, regardless of how close the raw analogy target vector is to the inputs. + final List result = model.analogy("man", "king", "woman", 4); + + assertEquals(2, result.size()); + assertFalse(result.stream().map(Neighbor::token) + .anyMatch(token -> List.of("man", "king", "woman").contains(token))); + } + + @Test + void testMostSimilarRejectsInvalidArguments(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertThrows(IllegalArgumentException.class, () -> model.mostSimilar(null, 1)); + assertThrows(IllegalArgumentException.class, () -> model.mostSimilar("king", 0)); + assertThrows(IllegalArgumentException.class, () -> model.mostSimilar("king", -1)); + } + + @Test + void testAnalogyRejectsInvalidArguments(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertThrows(IllegalArgumentException.class, () -> model.analogy(null, "king", "woman", 1)); + assertThrows(IllegalArgumentException.class, () -> model.analogy("man", null, "woman", 1)); + assertThrows(IllegalArgumentException.class, () -> model.analogy("man", "king", null, 1)); + assertThrows(IllegalArgumentException.class, () -> model.analogy("man", "king", "woman", 0)); + } + + @Test + void testSimilarityRejectsNullArguments(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + assertThrows(IllegalArgumentException.class, () -> model.similarity(null, "king")); + assertThrows(IllegalArgumentException.class, () -> model.similarity("king", null)); + } +} From b13f84999f08b9c2e8e439576136589aa72c7eb6 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 12:17:26 -0400 Subject: [PATCH 27/82] Add a JMH benchmark for StaticEmbeddingModel, matching the perf-1850-followup jmh profile Same opt-in jmh Maven profile pattern already used by opennlp-runtime (build-helper adds src/jmh/java as a test source root, jmh-core plus the annotation processor, activated only via -Pjmh; the default mvn verify is unaffected). Fixture is synthesized at the real minishlab/potion-base-8M scale (29,528 rows, 256 dimensions, both verified against the live model repo earlier) rather than downloaded, so the benchmark has no network dependency, but seeded with real English words so the benchmark sentences hit actual vocabulary entries instead of degenerating into all-[UNK] lookups. Forked run (2 forks x 10 iterations, the annotated configuration, not the quick-iteration main() override): embed (5 short sentences/op): 999,222.698 +/- 29,452.980 ops/s mostSimilarTop10 (full ~29.5k-row scan): 3,292.110 +/- 203.639 ops/s This is the JVM-only raw-throughput baseline the design doc calls for before any "faster than Python" claim; the concurrent-load comparison against a Python baseline is a separate, later benchmark. --- opennlp-extensions/opennlp-embeddings/pom.xml | 66 +++++++ .../StaticEmbeddingModelBenchmark.java | 171 ++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java diff --git a/opennlp-extensions/opennlp-embeddings/pom.xml b/opennlp-extensions/opennlp-embeddings/pom.xml index d1a724d80f..8ff56634d9 100644 --- a/opennlp-extensions/opennlp-embeddings/pom.xml +++ b/opennlp-extensions/opennlp-embeddings/pom.xml @@ -61,4 +61,70 @@ + + + jmh + + + org.openjdk.jmh + jmh-core + ${jmh.version} + test + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + test + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.1 + + + add-test-source + generate-test-sources + + add-test-source + + + + src/jmh/java + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + jmh-compile + test-compile + + testCompile + + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + + + + + + diff --git a/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java new file mode 100644 index 0000000000..03b5d75717 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +/** + * JMH benchmark for {@link StaticEmbeddingModel}, the raw-lookup-throughput number the module's + * design doc calls for before any "faster than Python" claim is made (a concurrent gRPC-traffic + * comparison against a Python baseline is a separate, later benchmark; this one is the JVM-only + * baseline). The fixture is sized to match {@code minishlab/potion-base-8M} (29,528 vocabulary + * rows, 256 dimensions), synthesized rather than downloaded so the benchmark has no network + * dependency, but seeded with real English words so the benchmark sentences tokenize into actual + * vocabulary hits rather than degenerating into all-unknown-token lookups. + */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 2) +@Measurement(iterations = 10, time = 2) +@Fork(2) +public class StaticEmbeddingModelBenchmark { + + // Matches minishlab/potion-base-8M's config.json (hidden_dim) and its reported total + // parameter count (7,559,168 / 256), verified against the real model repo, not guessed. + private static final int VOCAB_SIZE = 29_528; + private static final int DIMENSION = 256; + + private static final String[] REAL_WORDS = { + "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog", "she", "told", "me", "he", + "lived", "in", "wrote", "letter", "right", "away", "opennlp", "provides", "tools", "for", + "language", "processing", "driver", "got", "badly", "injured", "by", "accident", + }; + + private static final String[] SENTENCES = { + "The quick brown fox jumps over the lazy dog.", + "She told me he lived in Edinburgh.", + "I wrote him a letter right away.", + "OpenNLP provides tools for natural language processing.", + "The driver got badly injured by the accident.", + }; + + @State(Scope.Benchmark) + public static class ModelState { + + StaticEmbeddingModel model; + private Path tempDir; + + @Setup(Level.Trial) + public void load() throws IOException { + tempDir = Files.createTempDirectory("opennlp-embeddings-jmh"); + final Path vocabFile = writeVocab(tempDir); + final Path safetensorsFile = writeSafetensors(tempDir); + model = StaticEmbeddingModel.load(vocabFile, safetensorsFile, true, true); + } + + @TearDown(Level.Trial) + public void cleanup() throws IOException { + Files.deleteIfExists(tempDir.resolve("vocab.txt")); + Files.deleteIfExists(tempDir.resolve("model.safetensors")); + Files.deleteIfExists(tempDir); + } + + private static Path writeVocab(Path dir) throws IOException { + final List tokens = new ArrayList<>(VOCAB_SIZE); + tokens.add("[CLS]"); + tokens.add("[SEP]"); + tokens.add("[UNK]"); + for (final String word : REAL_WORDS) { + tokens.add(word); + } + while (tokens.size() < VOCAB_SIZE) { + tokens.add("tok" + (tokens.size() - REAL_WORDS.length - 3)); + } + final Path file = dir.resolve("vocab.txt"); + Files.write(file, tokens); + return file; + } + + private static Path writeSafetensors(Path dir) throws IOException { + final Random random = new Random(42); + final ByteBuffer buffer = ByteBuffer.allocate(VOCAB_SIZE * DIMENSION * 4) + .order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < VOCAB_SIZE * DIMENSION; i++) { + buffer.putFloat((random.nextFloat() - 0.5f) * 2f); + } + final byte[] data = buffer.array(); + final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + VOCAB_SIZE + "," + + DIMENSION + "],\"data_offsets\":[0," + data.length + "]}}"; + final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(headerBytes.length).array()); + out.write(headerBytes); + out.write(data); + final Path file = dir.resolve("model.safetensors"); + Files.write(file, out.toByteArray()); + return file; + } + } + + @Benchmark + @Threads(Threads.MAX) + public void embed(ModelState state, Blackhole blackhole) { + for (final String sentence : SENTENCES) { + blackhole.consume(state.model.embed(sentence)); + } + } + + @Benchmark + @Threads(Threads.MAX) + public void mostSimilarTop10(ModelState state, Blackhole blackhole) { + blackhole.consume(state.model.mostSimilar(SENTENCES[0], 10)); + } + + /** + * Quick local iteration only: {@code forks(0)} disables JVM fork isolation (unlike + * {@code mvn} with the {@code jmh} profile). Use the Maven-invoked configuration for + * publishable numbers. + */ + public static void main(String[] args) throws Exception { + final Options opt = new OptionsBuilder() + .include(StaticEmbeddingModelBenchmark.class.getSimpleName()) + .forks(0) + .warmupIterations(3) + .measurementIterations(5) + .build(); + new Runner(opt).run(); + } +} From c9381b8ca54f8ea365fd1de5f6d82a1cc0e2df2d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 13:23:02 -0400 Subject: [PATCH 28/82] Review pass on opennlp-embeddings: analogy exclusion fixes, faster neighbor scan, thread-safety hardening Two analogy() bugs fixed. Passing equal terms crashed with IllegalArgumentException("duplicate element") from Set.of; and the exclusion compared raw input strings against vocabulary tokens, so on an uncased model analogy("Man", "King", "Woman", k) handed "king" straight back as a result. Exclusion now folds the terms through the model's own tokenizer and excludes the resulting vocabulary rows, which makes it case- and accent-consistent with embed() and tolerant of equal or multiword terms. Both are pinned by new tests. Nearest-neighbor scan reworked around three observations: per-row L2 norms are constants of the model, so they are precomputed at load instead of recomputed (with a sqrt) for every row on every query; the top-K selection now uses a bounded min-heap over primitive parallel arrays instead of materializing and fully sorting one record per vocabulary row per query; and the special-token check is a precomputed boolean mask instead of per-row string hashing. The dot loop uses four accumulators because the JIT must not reorder floating-point additions and so cannot unroll the reduction itself. The zero-norm-row NaN guard is preserved and now has its own test. embed() drops an OptionalInt allocation per token (primitive -1 sentinel) and hoists the weight branch out of the accumulation loop. Forked JMH, same configuration and fixture as the recorded baseline: embed: 999,222 -> 1,041,654 ops/s (+4.2%) mostSimilarTop10: 3,292 -> 9,173 ops/s (2.79x) Thread safety reviewed and hardened: @ThreadSafe on StaticEmbeddingModel, SafetensorsFile, and WordPieceVocabulary; the class javadoc now documents why the one piece of global mutable state in the tokenizer chain (WhitespaceTokenizer.INSTANCE's keepNewLines flag) cannot affect results, since BERT basic tokenization replaces all whitespace with plain spaces before that split runs; and a new concurrency test runs 8 threads against one shared instance comparing every result to the single-threaded reference. --- .../opennlp/embeddings/SafetensorsFile.java | 6 +- .../embeddings/StaticEmbeddingModel.java | 246 +++++++++++++++--- .../embeddings/WordPieceVocabulary.java | 14 +- .../StaticEmbeddingModelConcurrencyTest.java | 127 +++++++++ .../StaticEmbeddingModelSimilarityTest.java | 64 +++++ 5 files changed, 418 insertions(+), 39 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 3e1bad57e3..6dd8075527 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -28,6 +28,8 @@ import java.util.Map; import java.util.Set; +import opennlp.tools.commons.ThreadSafe; + /** * Reads a safetensors file: an 8-byte * little-endian header length, a JSON header describing each tensor's dtype, shape, and byte @@ -42,8 +44,10 @@ * *

The whole file is read into memory up front (matching the project's existing bundled-data * readers), which is appropriate for the small (tens of megabytes) tables this module targets. - * Instances are immutable and safe for concurrent reads after construction.

+ * Instances are immutable and safe for concurrent reads after construction; every + * {@link #readFloat32(String)} call decodes into a fresh array the caller owns.

*/ +@ThreadSafe public final class SafetensorsFile { private static final int HEADER_LENGTH_PREFIX_BYTES = 8; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 51d4d8c45e..a6d417552f 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -17,12 +17,12 @@ package opennlp.embeddings; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Comparator; import java.util.List; -import java.util.OptionalInt; import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import opennlp.tools.commons.ThreadSafe; import opennlp.tools.tokenize.BertTokenizer; import opennlp.tools.tokenize.WordpieceTokenizer; @@ -43,13 +43,20 @@ * pooled vector is L2-normalized with an epsilon floor so a token-less input yields a zero * vector rather than a division by zero.

* - *

Instances are immutable and safe for concurrent {@link #embed(String)} calls after - * construction.

+ *

Thread safety. Instances are immutable and safe for concurrent use after + * construction: every field is final, the loaded arrays are never exposed or mutated, and the + * tokenizer chain holds no per-call state. The one piece of global mutable state in that chain, + * the {@code keepNewLines} flag on the {@code WhitespaceTokenizer.INSTANCE} singleton that + * {@link WordpieceTokenizer} splits with, cannot affect results here: BERT basic tokenization + * has already replaced every whitespace character, line breaks included, with plain spaces + * before that split runs, so the flag's only behavioral branch never triggers on this input.

*/ +@ThreadSafe public final class StaticEmbeddingModel { private static final float NORMALIZE_EPSILON = 1e-12f; private static final String WEIGHTS_TENSOR_NAME = "weights"; + private static final int[] NO_EXCLUDED_ROWS = new int[0]; // Never meaningful as a "similar word" result. private static final Set SPECIAL_TOKENS = Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN); @@ -61,10 +68,15 @@ public final class StaticEmbeddingModel { private final BertTokenizer tokenizer; private final boolean normalize; private final String unknownToken; + // Per-row L2 norms and the special-token mask are constants of the model, precomputed at + // load time so the nearest-neighbor scan does no per-row square-root or string hashing. + private final double[] rowNorms; + private final boolean[] specialRows; private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, WordPieceVocabulary vocabulary, BertTokenizer tokenizer, - boolean normalize, String unknownToken) { + boolean normalize, String unknownToken, double[] rowNorms, + boolean[] specialRows) { this.embeddings = embeddings; this.weights = weights; this.dimension = dimension; @@ -72,6 +84,8 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, this.tokenizer = tokenizer; this.normalize = normalize; this.unknownToken = unknownToken; + this.rowNorms = rowNorms; + this.specialRows = specialRows; } /** @@ -129,9 +143,27 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } } + final double[] rowNorms = new double[vocabulary.size()]; + for (int row = 0; row < rowNorms.length; row++) { + final int base = row * dimension; + double sumOfSquares = 0; + for (int d = 0; d < dimension; d++) { + final float value = embeddings[base + d]; + sumOfSquares += (double) value * value; + } + rowNorms[row] = Math.sqrt(sumOfSquares); + } + final boolean[] specialRows = new boolean[vocabulary.size()]; + for (final String special : SPECIAL_TOKENS) { + final int row = vocabulary.id(special); + if (row >= 0) { + specialRows[row] = true; + } + } + final BertTokenizer tokenizer = new BertTokenizer(vocabulary.tokens(), lowerCase); return new StaticEmbeddingModel(embeddings, weights, dimension, vocabulary, tokenizer, - normalize, WordpieceTokenizer.BERT_UNK_TOKEN); + normalize, WordpieceTokenizer.BERT_UNK_TOKEN, rowNorms, specialRows); } /** @@ -156,17 +188,23 @@ public float[] embed(String text) { if (unknownToken.equals(token)) { continue; } - final OptionalInt id = vocabulary.id(token); - if (id.isEmpty()) { + final int row = vocabulary.id(token); + if (row < 0) { throw new IllegalStateException("Tokenizer produced token '" + token + "' that is not in its own vocabulary; this indicates a tokenizer/vocabulary " + "construction bug, not an input problem"); } - final int row = id.getAsInt(); - final float weight = weights == null ? 1f : weights[row]; final int base = row * dimension; - for (int d = 0; d < dimension; d++) { - sum[d] += embeddings[base + d] * weight; + if (weights == null) { + for (int d = 0; d < dimension; d++) { + sum[d] += embeddings[base + d]; + } + } + else { + final float weight = weights[row]; + for (int d = 0; d < dimension; d++) { + sum[d] += embeddings[base + d] * weight; + } } pooledCount++; } @@ -236,7 +274,7 @@ public List mostSimilar(String text, int topK) { throw new IllegalArgumentException("Text must not be null"); } requirePositive(topK); - return nearestNeighbors(embed(text), topK, Set.of()); + return nearestNeighbors(embed(text), topK, NO_EXCLUDED_ROWS); } /** @@ -249,7 +287,10 @@ public List mostSimilar(String text, int topK) { * @param c The third term. Must not be {@code null}. * @param topK The maximum number of results. Must be at least 1. * @return Up to {@code topK} neighbors, most similar first, excluding the special tokens and - * any vocabulary token that exactly matches {@code a}, {@code b}, or {@code c}. + * every vocabulary token the three terms themselves tokenize to. The exclusion folds the + * terms exactly the way {@link #embed(String)} folds text, so on an uncased model a + * capitalized input excludes its lower-cased vocabulary row, and a multiword term + * excludes each of its word pieces. * @throws IllegalArgumentException Thrown if {@code a}, {@code b}, or {@code c} is * {@code null}, or {@code topK} is less than 1. */ @@ -271,7 +312,7 @@ public List analogy(String a, String b, String c, int topK) { for (int d = 0; d < dimension; d++) { target[d] = vb[d] - va[d] + vc[d]; } - return nearestNeighbors(target, topK, Set.of(a, b, c)); + return nearestNeighbors(target, topK, excludedRows(a, b, c)); } private static void requirePositive(int topK) { @@ -280,32 +321,83 @@ private static void requirePositive(int topK) { } } - private List nearestNeighbors(float[] query, int topK, Set exclude) { + // The vocabulary rows the given terms tokenize to, ascending and duplicate-free. Folding the + // terms through the model's own tokenizer (rather than comparing raw input strings against + // vocabulary tokens) is what makes the exclusion case- and accent-insensitive on uncased + // models, and it tolerates equal terms, which Set.of would reject as duplicates. + private int[] excludedRows(String... terms) { + final SortedSet rows = new TreeSet<>(); + for (final String term : terms) { + final String[] tokens = tokenizer.tokenize(term); + for (int i = 1; i < tokens.length - 1; i++) { + final String token = tokens[i]; + if (unknownToken.equals(token)) { + continue; + } + final int row = vocabulary.id(token); + if (row >= 0) { + rows.add(row); + } + } + } + final int[] sorted = new int[rows.size()]; + int i = 0; + for (final int row : rows) { + sorted[i++] = row; + } + return sorted; + } + + // The scan visits rows in ascending order and sortedExcludedRows is ascending, so exclusion + // is a single pointer that advances past each excluded row as the scan reaches it. + private List nearestNeighbors(float[] query, int topK, int[] sortedExcludedRows) { final double queryNorm = norm(query); if (queryNorm < NORMALIZE_EPSILON) { return List.of(); } - final List candidates = new ArrayList<>(vocabulary.size()); - for (int row = 0; row < vocabulary.size(); row++) { - final String token = vocabulary.token(row); - if (SPECIAL_TOKENS.contains(token) || exclude.contains(token)) { + final TopK best = new TopK(topK); + int nextExcluded = 0; + final int rowCount = rowNorms.length; + for (int row = 0; row < rowCount; row++) { + if (nextExcluded < sortedExcludedRows.length && sortedExcludedRows[nextExcluded] == row) { + nextExcluded++; + continue; + } + if (specialRows[row]) { + continue; + } + final double rowNorm = rowNorms[row]; + if (rowNorm < NORMALIZE_EPSILON) { + // A zero row has no direction; scored 0 rather than NaN from a 0/0 division. + best.offer(row, 0.0); continue; } final int base = row * dimension; - double dot = 0; - double rowNormSquared = 0; - for (int d = 0; d < dimension; d++) { - final float value = embeddings[base + d]; - dot += query[d] * value; - rowNormSquared += (double) value * value; + // Four accumulators because the JIT must not reorder floating-point additions and so + // cannot unroll this reduction itself; the split summation order is chosen deliberately. + double dot0 = 0; + double dot1 = 0; + double dot2 = 0; + double dot3 = 0; + int d = 0; + for (final int limit = dimension - 3; d < limit; d += 4) { + dot0 += query[d] * embeddings[base + d]; + dot1 += query[d + 1] * embeddings[base + d + 1]; + dot2 += query[d + 2] * embeddings[base + d + 2]; + dot3 += query[d + 3] * embeddings[base + d + 3]; + } + double dot = dot0 + dot1 + dot2 + dot3; + for (; d < dimension; d++) { + dot += query[d] * embeddings[base + d]; } - final double rowNorm = Math.sqrt(rowNormSquared); - final double similarity = rowNorm < NORMALIZE_EPSILON ? 0.0 : dot / (queryNorm * rowNorm); - candidates.add(new Neighbor(token, similarity)); + best.offer(row, dot / (queryNorm * rowNorm)); + } + final Neighbor[] ordered = new Neighbor[best.size()]; + for (int i = ordered.length - 1; i >= 0; i--) { + ordered[i] = new Neighbor(vocabulary.token(best.minRow()), best.minSimilarity()); + best.removeMin(); } - candidates.sort(Comparator.comparingDouble(Neighbor::similarity).reversed()); - return topK >= candidates.size() ? List.copyOf(candidates) - : List.copyOf(candidates.subList(0, topK)); + return List.of(ordered); } private static double cosineSimilarity(float[] a, float[] b) { @@ -328,4 +420,92 @@ private static double norm(float[] vector) { } return Math.sqrt(sumOfSquares); } + + /** + * A bounded selection of the {@code k} highest-similarity rows, kept as a min-heap over + * primitive parallel arrays: the root is always the weakest kept candidate, so a full scan + * decides most rows with one comparison against it and the selection allocates nothing per + * row (the previous implementation materialized and fully sorted one record per vocabulary + * row per query). + */ + private static final class TopK { + + private final double[] similarities; + private final int[] rows; + private int size; + + TopK(int capacity) { + this.similarities = new double[capacity]; + this.rows = new int[capacity]; + } + + void offer(int row, double similarity) { + if (size < similarities.length) { + int i = size++; + similarities[i] = similarity; + rows[i] = row; + while (i > 0) { + final int parent = (i - 1) >>> 1; + if (similarities[parent] <= similarities[i]) { + break; + } + swap(parent, i); + i = parent; + } + } + else if (similarity > similarities[0]) { + similarities[0] = similarity; + rows[0] = row; + siftDown(); + } + } + + int size() { + return size; + } + + int minRow() { + return rows[0]; + } + + double minSimilarity() { + return similarities[0]; + } + + void removeMin() { + size--; + similarities[0] = similarities[size]; + rows[0] = rows[size]; + siftDown(); + } + + private void siftDown() { + int i = 0; + while (true) { + final int left = 2 * i + 1; + final int right = left + 1; + int smallest = i; + if (left < size && similarities[left] < similarities[smallest]) { + smallest = left; + } + if (right < size && similarities[right] < similarities[smallest]) { + smallest = right; + } + if (smallest == i) { + return; + } + swap(i, smallest); + i = smallest; + } + } + + private void swap(int i, int j) { + final double similarity = similarities[i]; + similarities[i] = similarities[j]; + similarities[j] = similarity; + final int row = rows[i]; + rows[i] = rows[j]; + rows[j] = row; + } + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java index 10581b70ac..cdff03c833 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java @@ -24,9 +24,10 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.OptionalInt; import java.util.Set; +import opennlp.tools.commons.ThreadSafe; + /** * A BERT-style {@code vocab.txt} vocabulary: one token per line, the line number (0-based) is * the token's id. This is the same file format {@code bert-base-uncased} and the BGE family of @@ -36,6 +37,7 @@ * *

Immutable and safe for concurrent reads after construction.

*/ +@ThreadSafe final class WordPieceVocabulary { private final Map idByToken; @@ -92,17 +94,19 @@ Set tokens() { } /** - * Looks up a token's row id. + * Looks up a token's row id. Returns a primitive with a {@code -1} sentinel rather than an + * {@code OptionalInt} because this sits on the per-token hot path of + * {@link StaticEmbeddingModel#embed(String)}. * * @param token The token to look up. Must not be {@code null}. - * @return The token's id, or empty when the token is not in this vocabulary. + * @return The token's id, or {@code -1} when the token is not in this vocabulary. */ - OptionalInt id(String token) { + int id(String token) { if (token == null) { throw new IllegalArgumentException("Token must not be null"); } final Integer id = idByToken.get(token); - return id == null ? OptionalInt.empty() : OptionalInt.of(id); + return id == null ? -1 : id; } /** {@return the number of tokens in this vocabulary} */ diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java new file mode 100644 index 0000000000..1ccc23abdf --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A concurrency smoke test for the {@code @ThreadSafe} claim on {@link StaticEmbeddingModel}: + * one shared instance, many threads, every concurrent result compared against the + * single-threaded reference computed up front. All operations are deterministic, so any + * deviation under concurrency is a thread-safety defect by definition. Mirrors the + * {@code LexiconConcurrencyTest} pattern from the opennlp-wordnet module. + */ +class StaticEmbeddingModelConcurrencyTest { + + private static final int THREADS = 8; + private static final int ITERATIONS_PER_THREAD = 200; + + private static StaticEmbeddingModel loadFixture(Path dir) throws IOException { + final Path vocab = dir.resolve("vocab.txt"); + Files.write(vocab, + List.of("[CLS]", "[SEP]", "[UNK]", "king", "queen", "man", "woman", "apple")); + final float[][] rows = { + {0f, 0f}, {0f, 0f}, {0f, 0f}, + {3f, 3f}, {2f, 4f}, {2f, 1f}, {1f, 2f}, {-3f, -1f}, + }; + final ByteBuffer buffer = ByteBuffer.allocate(rows.length * 2 * 4) + .order(ByteOrder.LITTLE_ENDIAN); + for (final float[] row : rows) { + for (final float value : row) { + buffer.putFloat(value); + } + } + final byte[] data = buffer.array(); + final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + rows.length + + ",2],\"data_offsets\":[0," + data.length + "]}}"; + final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(headerBytes.length).array()); + out.write(headerBytes); + out.write(data); + final Path tensors = dir.resolve("model.safetensors"); + Files.write(tensors, out.toByteArray()); + return StaticEmbeddingModel.load(vocab, tensors, true, true); + } + + @Test + void testConcurrentUseMatchesSingleThreadedReference(@TempDir Path dir) throws Exception { + final StaticEmbeddingModel model = loadFixture(dir); + final float[] referenceEmbedding = model.embed("The King and Queen"); + final double referenceSimilarity = model.similarity("king", "queen"); + final List referenceNeighbors = model.mostSimilar("king", 3); + final List referenceAnalogy = model.analogy("man", "king", "woman", 2); + + final Queue problems = new ConcurrentLinkedQueue<>(); + final CountDownLatch start = new CountDownLatch(1); + final ExecutorService executor = Executors.newFixedThreadPool(THREADS); + try { + for (int t = 0; t < THREADS; t++) { + executor.submit(() -> { + try { + start.await(); + for (int i = 0; i < ITERATIONS_PER_THREAD; i++) { + if (!Arrays.equals(referenceEmbedding, model.embed("The King and Queen"))) { + problems.add("embed deviated from the single-threaded reference"); + } + if (referenceSimilarity != model.similarity("king", "queen")) { + problems.add("similarity deviated from the single-threaded reference"); + } + if (!referenceNeighbors.equals(model.mostSimilar("king", 3))) { + problems.add("mostSimilar deviated from the single-threaded reference"); + } + if (!referenceAnalogy.equals(model.analogy("man", "king", "woman", 2))) { + problems.add("analogy deviated from the single-threaded reference"); + } + } + } + catch (Exception e) { + problems.add("Unexpected exception: " + e); + } + }); + } + start.countDown(); + executor.shutdown(); + assertTrue(executor.awaitTermination(2, TimeUnit.MINUTES), + "Concurrent workers did not finish in time"); + } + finally { + executor.shutdownNow(); + } + assertTrue(problems.isEmpty(), () -> "Thread-safety violations: " + problems); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java index 103788c679..b208de352c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java @@ -184,6 +184,70 @@ void testAnalogyExcludesItsOwnInputTerms(@TempDir Path dir) throws IOException { .anyMatch(token -> List.of("man", "king", "woman").contains(token))); } + @Test + void testAnalogyToleratesEqualTerms(@TempDir Path dir) throws IOException { + // A duplicate term used to crash with IllegalArgumentException("duplicate element") from + // Set.of before the exclusion moved to tokenized rows. b - a + c with a == b is just c's + // vector, so with man and woman excluded the exactly collinear queen must win. + final StaticEmbeddingModel model = load(dir); + + final List result = model.analogy("man", "man", "woman", 2); + + assertEquals("queen", result.get(0).token()); + assertEquals(1.0, result.get(0).similarity(), 1e-5); + } + + @Test + void testAnalogyExclusionFoldsLikeEmbed(@TempDir Path dir) throws IOException { + // On an uncased model, capitalized inputs must exclude their lower-cased vocabulary rows. + // Before the fix the exclusion compared raw input strings, so "King" failed to exclude + // "king" and the analogy handed an input term back as a result. + final StaticEmbeddingModel model = load(dir); + + final List result = model.analogy("Man", "King", "Woman", 4); + + assertEquals(2, result.size()); + assertEquals("queen", result.get(0).token()); + assertFalse(result.stream().map(Neighbor::token) + .anyMatch(token -> List.of("man", "king", "woman").contains(token))); + } + + @Test + void testZeroVectorRowScoresZeroNotNaN(@TempDir Path dir) throws IOException { + // A non-special all-zero row has no direction; it must score exactly 0.0, not the NaN a + // naive 0/0 cosine would produce. + final Path vocab = dir.resolve("zero-vocab.txt"); + Files.write(vocab, List.of("[CLS]", "[SEP]", "[UNK]", "a", "zero")); + final float[][] rows = {{0f, 0f}, {0f, 0f}, {0f, 0f}, {1f, 0f}, {0f, 0f}}; + final ByteBuffer buffer = ByteBuffer.allocate(rows.length * 2 * 4) + .order(ByteOrder.LITTLE_ENDIAN); + for (final float[] row : rows) { + for (final float value : row) { + buffer.putFloat(value); + } + } + final byte[] data = buffer.array(); + final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + rows.length + + ",2],\"data_offsets\":[0," + data.length + "]}}"; + final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) + .putLong(headerBytes.length).array()); + out.write(headerBytes); + out.write(data); + final Path tensors = dir.resolve("zero-model.safetensors"); + Files.write(tensors, out.toByteArray()); + final StaticEmbeddingModel model = StaticEmbeddingModel.load(vocab, tensors, true, false); + + final List result = model.mostSimilar("a", 5); + + assertEquals(2, result.size()); + assertEquals("a", result.get(0).token()); + assertEquals("zero", result.get(1).token()); + assertEquals(0.0, result.get(1).similarity()); + assertTrue(result.stream().allMatch(neighbor -> Double.isFinite(neighbor.similarity()))); + } + @Test void testMostSimilarRejectsInvalidArguments(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = load(dir); From a919cebaf2a3ab238915519e77caf44bc3ac299f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 9 Jul 2026 16:19:19 -0400 Subject: [PATCH 29/82] Stream safetensors tensor data with positional reads Replaces the whole-file byte[] (capped at 2 GB by Java's int-indexed arrays, and failing as an opaque OutOfMemoryError beyond it) with positional FileChannel reads: the header is read eagerly, tensor data streams straight into the caller's float[] through a reused 1 MB chunk. File size is now unlimited; the remaining ceiling is per decoded tensor (a float[] holds at most ~2.1 billion elements) and is checked with a clear message. Peak load memory drops since file bytes and the decoded array no longer coexist. A file truncated between read() and readFloat32() fails loud instead of returning partial data. --- .../opennlp/embeddings/SafetensorsFile.java | 165 +++++++++++++----- .../embeddings/SafetensorsFileTest.java | 33 ++++ 2 files changed, 150 insertions(+), 48 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 6dd8075527..3fd36d2b0a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -20,9 +20,11 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -42,31 +44,45 @@ * one cannot execute arbitrary code. No hardening beyond ordinary malformed-input handling is * needed.

* - *

The whole file is read into memory up front (matching the project's existing bundled-data - * readers), which is appropriate for the small (tens of megabytes) tables this module targets. - * Instances are immutable and safe for concurrent reads after construction; every - * {@link #readFloat32(String)} call decodes into a fresh array the caller owns.

+ *

Only the header is read eagerly; tensor data is streamed straight into the caller's array + * with positional reads when requested, so the file size is not limited by Java's int-indexed + * arrays. The remaining ceiling is per tensor, not per file: one decoded {@code float[]} holds + * at most {@link Integer#MAX_VALUE} - 8 elements, and {@link #readFloat32(String)} checks that + * explicitly. The file must stay in place and unchanged between {@link #read(Path)} and later + * {@link #readFloat32(String)} calls; a file truncated in between fails loud rather than + * returning partial data.

+ * + *

Instances are immutable and safe for concurrent use: every {@link #readFloat32(String)} + * call opens its own channel and decodes into a fresh array the caller owns.

*/ @ThreadSafe public final class SafetensorsFile { private static final int HEADER_LENGTH_PREFIX_BYTES = 8; - private final byte[] bytes; + // Positional-read chunk size, a multiple of Float.BYTES so every filled chunk decodes to + // whole floats. + private static final int READ_CHUNK_BYTES = 1 << 20; + + // The JVM refuses array allocations slightly below Integer.MAX_VALUE; the exact headroom is + // implementation-specific, 8 is the commonly reserved amount. + private static final long MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8; + + private final Path file; private final long dataStart; private final Map tensorsByName; private final Map metadata; - private SafetensorsFile(byte[] bytes, long dataStart, Map tensorsByName, + private SafetensorsFile(Path file, long dataStart, Map tensorsByName, Map metadata) { - this.bytes = bytes; + this.file = file; this.dataStart = dataStart; this.tensorsByName = tensorsByName; this.metadata = metadata; } /** - * Reads a safetensors file. + * Reads a safetensors file's header. * * @param file The file to read. Must not be {@code null} and must exist. * @return The parsed file, with every tensor's metadata resolved and validated against the @@ -82,42 +98,50 @@ public static SafetensorsFile read(Path file) { if (!Files.isRegularFile(file)) { throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); } - final byte[] bytes; - try { - bytes = Files.readAllBytes(file); + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { + final long fileSize = channel.size(); + if (fileSize < HEADER_LENGTH_PREFIX_BYTES) { + throw new IllegalArgumentException( + "File " + file + " is too short to be a safetensors file: " + fileSize + " bytes"); + } + final ByteBuffer prefix = ByteBuffer.allocate(HEADER_LENGTH_PREFIX_BYTES) + .order(ByteOrder.LITTLE_ENDIAN); + readFully(channel, prefix, 0, file); + final long headerLength = prefix.flip().getLong(); + if (headerLength < 0 || headerLength > fileSize - HEADER_LENGTH_PREFIX_BYTES) { + throw new IllegalArgumentException("File " + file + " declares a header length of " + + headerLength + ", which does not fit in a file of " + fileSize + " bytes"); + } + if (headerLength > MAX_ARRAY_LENGTH) { + throw new IllegalArgumentException("File " + file + " declares a header length of " + + headerLength + " bytes, too large to decode as a single JSON string"); + } + final ByteBuffer headerBytes = ByteBuffer.allocate((int) headerLength); + readFully(channel, headerBytes, HEADER_LENGTH_PREFIX_BYTES, file); + final String headerJson = new String(headerBytes.array(), StandardCharsets.UTF_8); + final SafetensorsHeaderParser.Result parsed = SafetensorsHeaderParser.parse(headerJson); + final long dataStart = HEADER_LENGTH_PREFIX_BYTES + headerLength; + final long dataLength = fileSize - dataStart; + final Map tensorsByName = + new LinkedHashMap<>(parsed.tensors().size() * 2); + for (final TensorInfo tensor : parsed.tensors()) { + if (tensor.dataOffsetBegin() < 0 || tensor.dataOffsetEnd() < tensor.dataOffsetBegin() + || tensor.dataOffsetEnd() > dataLength) { + throw new IllegalArgumentException("File " + file + " tensor '" + tensor.name() + + "' has a data range [" + tensor.dataOffsetBegin() + ", " + tensor.dataOffsetEnd() + + ") that does not fit in the file"); + } + if (tensorsByName.putIfAbsent(tensor.name(), tensor) != null) { + throw new IllegalArgumentException( + "File " + file + " declares tensor '" + tensor.name() + "' more than once"); + } + } + return new SafetensorsFile(file, dataStart, Collections.unmodifiableMap(tensorsByName), + Collections.unmodifiableMap(parsed.metadata())); } catch (IOException e) { throw new UncheckedIOException("Unable to read safetensors file " + file, e); } - if (bytes.length < HEADER_LENGTH_PREFIX_BYTES) { - throw new IllegalArgumentException( - "File " + file + " is too short to be a safetensors file: " + bytes.length + " bytes"); - } - final long headerLength = ByteBuffer.wrap(bytes, 0, HEADER_LENGTH_PREFIX_BYTES) - .order(ByteOrder.LITTLE_ENDIAN).getLong(); - final long dataStart = (long) HEADER_LENGTH_PREFIX_BYTES + headerLength; - if (headerLength < 0 || dataStart > bytes.length) { - throw new IllegalArgumentException("File " + file + " declares a header length of " - + headerLength + ", which does not fit in a file of " + bytes.length + " bytes"); - } - final String headerJson = new String(bytes, HEADER_LENGTH_PREFIX_BYTES, (int) headerLength, - StandardCharsets.UTF_8); - final SafetensorsHeaderParser.Result parsed = SafetensorsHeaderParser.parse(headerJson); - final Map tensorsByName = new LinkedHashMap<>(parsed.tensors().size() * 2); - for (final TensorInfo tensor : parsed.tensors()) { - if (tensor.dataOffsetBegin() < 0 || tensor.dataOffsetEnd() < tensor.dataOffsetBegin() - || dataStart + tensor.dataOffsetEnd() > bytes.length) { - throw new IllegalArgumentException("File " + file + " tensor '" + tensor.name() - + "' has a data range [" + tensor.dataOffsetBegin() + ", " + tensor.dataOffsetEnd() - + ") that does not fit in the file"); - } - if (tensorsByName.putIfAbsent(tensor.name(), tensor) != null) { - throw new IllegalArgumentException( - "File " + file + " declares tensor '" + tensor.name() + "' more than once"); - } - } - return new SafetensorsFile(bytes, dataStart, Collections.unmodifiableMap(tensorsByName), - Collections.unmodifiableMap(parsed.metadata())); } /** {@return the names of every tensor declared in the header, in header order} */ @@ -146,12 +170,15 @@ public TensorInfo tensorInfo(String name) { } /** - * Decodes a {@code F32} tensor's data. + * Decodes a {@code F32} tensor's data, streaming it from the file. * * @param name The tensor's name. Must not be {@code null}. * @return The tensor's elements in row-major (shape outermost-first) order. * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, not a tensor in - * this file, or not declared with dtype {@code F32}. + * this file, not declared with dtype {@code F32}, or larger than a Java array can hold. + * @throws IllegalStateException Thrown if the file has been truncated since + * {@link #read(Path)} validated the tensor's byte range. + * @throws UncheckedIOException Thrown if reading the file fails. */ public float[] readFloat32(String name) { final TensorInfo info = tensorInfo(name); @@ -160,17 +187,59 @@ public float[] readFloat32(String name) { "Tensor '" + name + "' has dtype " + info.dtype() + ", not F32"); } final long elementCount = info.elementCount(); + if (elementCount < 0 || elementCount > MAX_ARRAY_LENGTH) { + throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount + + " elements, more than a Java array can hold (" + MAX_ARRAY_LENGTH + + "); decoding to a float[] is capped there"); + } final long byteLength = info.dataOffsetEnd() - info.dataOffsetBegin(); - if (byteLength != elementCount * 4L) { + if (byteLength != elementCount * Float.BYTES) { throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount + " F32 elements but its data range is " + byteLength + " bytes"); } final float[] values = new float[(int) elementCount]; - final ByteBuffer buffer = ByteBuffer.wrap(bytes, - (int) (dataStart + info.dataOffsetBegin()), (int) byteLength) - .order(ByteOrder.LITTLE_ENDIAN); - buffer.asFloatBuffer().get(values); - return values; + // When the build baseline reaches JDK 22+, this loop can become a single MemorySegment.copy + // out of a FileChannel.map'd segment (long-indexed, deterministic unmap via Arena); on the + // JDK 21 baseline java.lang.foreign is still a preview API, so positional reads are the + // portable way past the 2 GB byte[]/ByteBuffer ceiling. + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { + final ByteBuffer chunk = ByteBuffer.allocate((int) Math.min(READ_CHUNK_BYTES, byteLength)) + .order(ByteOrder.LITTLE_ENDIAN); + long position = dataStart + info.dataOffsetBegin(); + int decoded = 0; + while (decoded < values.length) { + chunk.clear(); + final long remainingBytes = byteLength - (long) decoded * Float.BYTES; + if (remainingBytes < chunk.capacity()) { + chunk.limit((int) remainingBytes); + } + readFully(channel, chunk, position, file); + chunk.flip(); + final int floats = chunk.remaining() / Float.BYTES; + chunk.asFloatBuffer().get(values, decoded, floats); + decoded += floats; + position += (long) floats * Float.BYTES; + } + return values; + } + catch (IOException e) { + throw new UncheckedIOException("Unable to read tensor '" + name + "' from " + file, e); + } + } + + // Fills the buffer with bytes starting at the given file position; fails loud if the file + // ends first, which can only happen when the file shrank after read(Path) validated ranges + // against its length. + private static void readFully(FileChannel channel, ByteBuffer buffer, long position, Path file) + throws IOException { + while (buffer.hasRemaining()) { + final int read = channel.read(buffer, position + buffer.position()); + if (read < 0) { + throw new IllegalStateException("File " + file + " ended at byte " + + (position + buffer.position()) + + "; it has been truncated since its header was read"); + } + } } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java index d7f55dfa30..21a6cb30d6 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java @@ -258,4 +258,37 @@ void testRejectsUnterminatedString(@TempDir Path dir) throws IOException { assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); } + + @Test + void testRejectsTensorLargerThanAJavaArray(@TempDir Path dir) throws IOException { + // 2_000_000 * 2_000 = 4 billion elements, over the float[] ceiling. The bogus small data + // range keeps the file tiny; the array-ceiling check fires before the range-mismatch check + // because it subsumes it for tensors this large. + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2000000,2000]," + + "\"data_offsets\":[0,4]}}"; + final Path file = writeFile(dir, "model.safetensors", header, new byte[] {1, 2, 3, 4}); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("w")); + assertTrue(e.getMessage().contains("more than a Java array can hold")); + } + + @Test + void testFailsLoudWhenTheFileIsTruncatedAfterRead(@TempDir Path dir) throws IOException { + // Tensor data is streamed on demand rather than held in memory, so a file that shrinks + // between read() and readFloat32() must fail loud, not return partial data. + final byte[] data = floatsToLittleEndianBytes(1f, 2f); + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2]," + + "\"data_offsets\":[0," + data.length + "]}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + writeFile(dir, "model.safetensors", header, floatsToLittleEndianBytes(1f)); + + final IllegalStateException e = + assertThrows(IllegalStateException.class, () -> parsed.readFloat32("w")); + assertTrue(e.getMessage().contains("truncated")); + } } From 795734ba1bfe274a4f36cb73d2284ed9a2fbdfc4 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 9 Jul 2026 16:21:21 -0400 Subject: [PATCH 30/82] Unit-test SafetensorsHeaderParser directly, reject trailing header garbage Writing the direct tests surfaced one gap: parseTop stopped at the closing brace and silently ignored anything after it. Trailing whitespace stays legal (writers space-pad the header to align the data section), any other trailing content now fails loud. --- .../embeddings/SafetensorsHeaderParser.java | 11 ++ .../SafetensorsHeaderParserTest.java | 158 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java index 6c05072d7b..5ec7f5d57a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java @@ -65,6 +65,7 @@ private Result parseTop() { skipWhitespace(); if (peek() == '}') { position++; + requireEnd(); return new Result(tensors, metadata); } while (true) { @@ -89,9 +90,19 @@ private Result parseTop() { } throw malformed("Expected ',' or '}' after a header entry, got '" + next + "'"); } + requireEnd(); return new Result(tensors, metadata); } + // Trailing whitespace is legal (writers space-pad the header to align the data section), but + // any other trailing content means the declared header length and the JSON disagree. + private void requireEnd() { + skipWhitespace(); + if (position < text.length()) { + throw malformed("Trailing content after the header object"); + } + } + private TensorInfo parseTensorInfo(String name) { expect('{'); String dtype = null; diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java new file mode 100644 index 0000000000..e9ebf71538 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Direct tests for {@link SafetensorsHeaderParser}, complementing the indirect coverage in + * {@link SafetensorsFileTest}: the file-level tests exercise headers as whole files, these pin + * the parser's own contract, its error offsets, and every malformed-input branch. + */ +class SafetensorsHeaderParserTest { + + @Test + void testParsesTensorsInHeaderOrder() { + final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( + "{\"beta\":{\"dtype\":\"F32\",\"shape\":[2,3],\"data_offsets\":[0,24]}," + + "\"alpha\":{\"dtype\":\"I64\",\"shape\":[],\"data_offsets\":[24,32]}}"); + + assertEquals(2, result.tensors().size()); + final TensorInfo beta = result.tensors().get(0); + assertEquals("beta", beta.name()); + assertEquals("F32", beta.dtype()); + assertArrayEquals(new int[] {2, 3}, beta.shape()); + assertEquals(0, beta.dataOffsetBegin()); + assertEquals(24, beta.dataOffsetEnd()); + final TensorInfo alpha = result.tensors().get(1); + assertEquals("alpha", alpha.name()); + assertArrayEquals(new int[0], alpha.shape()); + assertEquals(1, alpha.elementCount()); + assertTrue(result.metadata().isEmpty()); + } + + @Test + void testParsesAnEmptyHeader() { + final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse("{}"); + + assertTrue(result.tensors().isEmpty()); + assertTrue(result.metadata().isEmpty()); + } + + @Test + void testParsesAMetadataOnlyHeader() { + final SafetensorsHeaderParser.Result result = + SafetensorsHeaderParser.parse("{\"__metadata__\":{\"format\":\"pt\"}}"); + + assertTrue(result.tensors().isEmpty()); + assertEquals("pt", result.metadata().get("format")); + } + + @Test + void testDecodesEveryEscapeSequence() { + final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( + "{\"__metadata__\":{\"note\":\"\\\"\\\\\\/\\b\\f\\n\\r\\t\\u0041\"}}"); + + assertEquals("\"\\/\b\f\n\r\tA", result.metadata().get("note")); + } + + @Test + void testSkipsUnknownFieldsOfEveryValueType() { + // Fields safetensors may add over time must not break the reader: nested objects, arrays, + // floating-point numbers, booleans, null, and strings are all skipped structurally. + final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( + "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]," + + "\"future\":{\"nested\":[1,-2.5e3,true,false,null,\"s\",{\"deep\":[]}]}}}"); + + assertEquals(List.of("w"), result.tensors().stream().map(TensorInfo::name).toList()); + } + + @Test + void testToleratesTrailingWhitespacePadding() { + // Writers space-pad the header so the data section starts aligned; padding is part of the + // declared header length and must parse cleanly. + final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( + "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}} "); + + assertEquals(1, result.tensors().size()); + } + + @Test + void testRejectsTrailingGarbage() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> SafetensorsHeaderParser.parse("{} x")); + assertTrue(e.getMessage().contains("Trailing content")); + } + + @Test + void testRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(null)); + } + + @ParameterizedTest + @ValueSource(strings = { + // unterminated string + "{\"w", + // unknown escape + "{\"a\\x\":{}}", + // truncated \_u escape (split so the Java lexer does not see a \_u sequence) + "{\"a\\" + "u00", + // malformed \_u escape + "{\"a\\" + "uZZZZ\":{}}", + // missing colon + "{\"w\" 1}", + // empty tensor object + "{\"w\":{}}", + // missing dtype + "{\"w\":{\"shape\":[1],\"data_offsets\":[0,4]}}", + // missing shape + "{\"w\":{\"dtype\":\"F32\",\"data_offsets\":[0,4]}}", + // missing data_offsets + "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]}}", + // data_offsets arity 1 + "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0]}}", + // data_offsets arity 3 + "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4,8]}}", + // negative shape dimension + "{\"w\":{\"dtype\":\"F32\",\"shape\":[-1],\"data_offsets\":[0,4]}}", + // shape dimension over int range + "{\"w\":{\"dtype\":\"F32\",\"shape\":[4294967296],\"data_offsets\":[0,4]}}", + // non-numeric array element + "{\"w\":{\"dtype\":\"F32\",\"shape\":[\"x\"],\"data_offsets\":[0,4]}}", + // number too large for long + "{\"w\":{\"dtype\":\"F32\",\"shape\":[99999999999999999999],\"data_offsets\":[0,4]}}", + // bare value instead of an object + "42", + // truncated after a tensor entry + "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}" + }) + void testRejectsMalformedHeaders(String header) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> SafetensorsHeaderParser.parse(header)); + assertTrue(e.getMessage().contains("Malformed safetensors header at offset"), + () -> "Message should carry the offset, got: " + e.getMessage()); + } +} From dcd15e95831c1fbc5d0c2d712118dc1b370bcb18 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 9 Jul 2026 16:25:11 -0400 Subject: [PATCH 31/82] Add StaticEmbeddingModel.load(modelDirectory) resolving switches from the model configs Reads normalize from config.json and do_lower_case from tokenizer_config.json (field names verified against published Model2Vec-family releases), so callers no longer have to know a model's switches to load it. The JSON scanning primitives move from SafetensorsHeaderParser into a shared JsonCursor; FlatJsonFields reads just the top-level booleans and skips everything else structurally. A strip_accents explicitly set against do_lower_case is rejected rather than silently mis-tokenized: the single lower-case switch follows the BERT convention of stripping accents exactly when lower-casing, and the error points at the explicit overload for deliberate choices. --- .../opennlp/embeddings/FlatJsonFields.java | 105 +++++++ .../java/opennlp/embeddings/JsonCursor.java | 227 ++++++++++++++ .../embeddings/SafetensorsHeaderParser.java | 295 ++++-------------- .../embeddings/StaticEmbeddingModel.java | 77 +++++ .../embeddings/FlatJsonFieldsTest.java | 119 +++++++ .../embeddings/StaticEmbeddingModelTest.java | 86 +++++ 6 files changed, 673 insertions(+), 236 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java new file mode 100644 index 0000000000..6ba963dbd4 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Reads single top-level fields out of a small flat JSON configuration file (a model's + * {@code config.json} or {@code tokenizer_config.json}) without a JSON library dependency, + * sharing {@link JsonCursor}'s scanning primitives with the safetensors header parser. Only + * what the model-directory loader needs is implemented: top-level boolean look-ups. Every + * other field, of any type and nesting, is skipped structurally, and nested occurrences of the + * looked-up name never match (a top-level field is what the configuration formats define). + */ +final class FlatJsonFields { + + private FlatJsonFields() { + } + + /** + * Reads one top-level boolean field from a JSON object file. + * + * @param file The JSON file, a single top-level object. Must not be {@code null} and must + * exist. + * @param field The top-level field name to read. Must not be {@code null}. + * @return The field's value, or {@code null} when the field is absent or explicitly JSON + * {@code null} (the formats treat those the same: fall back to the default). + * @throws IllegalArgumentException Thrown if the file is not a well-formed JSON object, the + * field appears more than once, or its value is neither a boolean nor {@code null}. + * @throws UncheckedIOException Thrown if reading the file fails. + */ + static Boolean topLevelBoolean(Path file, String field) { + final String json; + try { + json = Files.readString(file); + } + catch (IOException e) { + throw new UncheckedIOException("Unable to read " + file, e); + } + final JsonCursor cursor = new JsonCursor(json, file.getFileName().toString()); + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + Boolean value = null; + boolean seen = false; + if (cursor.peek() == '}') { + cursor.consume(); + } + else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if (field.equals(key)) { + if (seen) { + throw cursor.malformed("Field '" + field + "' appears more than once"); + } + seen = true; + if (cursor.consumeLiteral("true")) { + value = Boolean.TRUE; + } + else if (cursor.consumeLiteral("false")) { + value = Boolean.FALSE; + } + else if (!cursor.consumeLiteral("null")) { + throw cursor.malformed("Field '" + field + "' must be a boolean or null"); + } + } + else { + cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a field, got '" + next + "'"); + } + } + cursor.requireEnd("Trailing content after the top-level object"); + return value; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java new file mode 100644 index 0000000000..6947950d0a --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +/** + * Cursor primitives shared by this package's purpose-built JSON readers + * ({@link SafetensorsHeaderParser}, {@link FlatJsonFields}): string and integer scalars, + * literals, whitespace, and skipping one value of any type. Deliberately not a general JSON + * library: no floating-point decoding, no document model; each reader drives the cursor over + * its own known-shape input and fails loud on anything else, with the input's name and the + * offending offset in every message. + */ +final class JsonCursor { + + private final String text; + private final String inputName; + private int position; + + /** + * @param text The JSON text to scan. Must not be {@code null}. + * @param inputName What the text is (for error messages), e.g. {@code "safetensors header"} + * or a file name. + */ + JsonCursor(String text, String inputName) { + this.text = text; + this.inputName = inputName; + } + + void skipWhitespace() { + while (position < text.length() && Character.isWhitespace(text.charAt(position))) { + position++; + } + } + + char peek() { + if (position >= text.length()) { + throw malformed("Unexpected end of input"); + } + return text.charAt(position); + } + + char consume() { + final char c = peek(); + position++; + return c; + } + + void expect(char c) { + final char actual = consume(); + if (actual != c) { + throw malformed("Expected '" + c + "', got '" + actual + "'"); + } + } + + /** Consumes the given literal (e.g. {@code "true"}) if it starts here; returns whether. */ + boolean consumeLiteral(String literal) { + if (text.startsWith(literal, position)) { + position += literal.length(); + return true; + } + return false; + } + + /** Requires the rest of the input to be whitespace only. */ + void requireEnd(String message) { + skipWhitespace(); + if (position < text.length()) { + throw malformed(message); + } + } + + String parseString() { + expect('"'); + final StringBuilder value = new StringBuilder(); + while (true) { + if (position >= text.length()) { + throw malformed("Unterminated string"); + } + final char c = text.charAt(position++); + if (c == '"') { + return value.toString(); + } + if (c == '\\') { + value.append(parseEscape()); + } + else { + value.append(c); + } + } + } + + private char parseEscape() { + if (position >= text.length()) { + throw malformed("Unterminated escape sequence"); + } + final char escape = text.charAt(position++); + return switch (escape) { + case '"' -> '"'; + case '\\' -> '\\'; + case '/' -> '/'; + case 'b' -> '\b'; + case 'f' -> '\f'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 't' -> '\t'; + case 'u' -> parseUnicodeEscape(); + default -> throw malformed("Unknown escape sequence: \\" + escape); + }; + } + + private char parseUnicodeEscape() { + if (position + 4 > text.length()) { + throw malformed("Truncated \\u escape sequence"); + } + final String hex = text.substring(position, position + 4); + position += 4; + try { + return (char) Integer.parseInt(hex, 16); + } + catch (NumberFormatException e) { + throw malformed("Malformed \\u escape sequence: " + hex); + } + } + + long parseLong() { + final int start = position; + if (peek() == '-') { + position++; + } + if (position >= text.length() || !Character.isDigit(text.charAt(position))) { + throw malformed("Expected an integer"); + } + while (position < text.length() && Character.isDigit(text.charAt(position))) { + position++; + } + try { + return Long.parseLong(text.substring(start, position)); + } + catch (NumberFormatException e) { + throw malformed("Malformed integer: " + text.substring(start, position)); + } + } + + // Skips one JSON value of any type (string, number, array, object, true/false/null); used + // for fields a reader does not care about, so unknown additions never break it. + void skipValue() { + skipWhitespace(); + final char c = peek(); + if (c == '"') { + parseString(); + } + else if (c == '[') { + position++; + skipWhitespace(); + if (peek() != ']') { + while (true) { + skipValue(); + skipWhitespace(); + final char next = consume(); + if (next == ',') { + skipWhitespace(); + continue; + } + if (next == ']') { + return; + } + throw malformed("Expected ',' or ']' while skipping an array, got '" + next + "'"); + } + } + position++; + } + else if (c == '{') { + position++; + skipWhitespace(); + if (peek() != '}') { + while (true) { + skipWhitespace(); + parseString(); + skipWhitespace(); + expect(':'); + skipValue(); + skipWhitespace(); + final char next = consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return; + } + throw malformed("Expected ',' or '}' while skipping an object, got '" + next + "'"); + } + } + position++; + } + else if (c == '-' || Character.isDigit(c)) { + position++; + while (position < text.length() && "0123456789.eE+-".indexOf(text.charAt(position)) >= 0) { + position++; + } + } + else if (consumeLiteral("true") || consumeLiteral("false") || consumeLiteral("null")) { + // consumed, nothing to record + } + else { + throw malformed("Unexpected character while skipping a value: '" + c + "'"); + } + } + + IllegalArgumentException malformed(String message) { + return new IllegalArgumentException( + "Malformed " + inputName + " at offset " + position + ": " + message); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java index 5ec7f5d57a..8a6b0989e3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java @@ -27,17 +27,17 @@ * {@code data_offsets} record, plus an optional {@code __metadata__} string map), not a * general-purpose JSON parser: no floating-point numbers, no arbitrary nesting depth, no * comments. This is the same discipline used by every other data-file cursor parser in the - * project (no regular expressions, fail loud on malformed input). + * project (no regular expressions, fail loud on malformed input); the scanning primitives are + * shared with {@link FlatJsonFields} through {@link JsonCursor}. */ final class SafetensorsHeaderParser { private static final String METADATA_KEY = "__metadata__"; - private final String text; - private int position; + private final JsonCursor cursor; private SafetensorsHeaderParser(String text) { - this.text = text; + this.cursor = new JsonCursor(text, "safetensors header"); } /** @@ -60,35 +60,35 @@ static Result parse(String headerJson) { private Result parseTop() { final List tensors = new ArrayList<>(); Map metadata = Map.of(); - skipWhitespace(); - expect('{'); - skipWhitespace(); - if (peek() == '}') { - position++; + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + if (cursor.peek() == '}') { + cursor.consume(); requireEnd(); return new Result(tensors, metadata); } while (true) { - skipWhitespace(); - final String key = parseString(); - skipWhitespace(); - expect(':'); - skipWhitespace(); + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); if (METADATA_KEY.equals(key)) { metadata = parseStringMap(); } else { tensors.add(parseTensorInfo(key)); } - skipWhitespace(); - final char next = consume(); + cursor.skipWhitespace(); + final char next = cursor.consume(); if (next == ',') { continue; } if (next == '}') { break; } - throw malformed("Expected ',' or '}' after a header entry, got '" + next + "'"); + throw cursor.malformed("Expected ',' or '}' after a header entry, got '" + next + "'"); } requireEnd(); return new Result(tensors, metadata); @@ -97,82 +97,79 @@ private Result parseTop() { // Trailing whitespace is legal (writers space-pad the header to align the data section), but // any other trailing content means the declared header length and the JSON disagree. private void requireEnd() { - skipWhitespace(); - if (position < text.length()) { - throw malformed("Trailing content after the header object"); - } + cursor.requireEnd("Trailing content after the header object"); } private TensorInfo parseTensorInfo(String name) { - expect('{'); + cursor.expect('{'); String dtype = null; int[] shape = null; long dataOffsetBegin = -1; long dataOffsetEnd = -1; - skipWhitespace(); - while (peek() != '}') { - skipWhitespace(); - final String field = parseString(); - skipWhitespace(); - expect(':'); - skipWhitespace(); + cursor.skipWhitespace(); + while (cursor.peek() != '}') { + cursor.skipWhitespace(); + final String field = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); switch (field) { - case "dtype" -> dtype = parseString(); + case "dtype" -> dtype = cursor.parseString(); case "shape" -> shape = parseIntArray(); case "data_offsets" -> { final long[] offsets = parseLongArray(); if (offsets.length != 2) { - throw malformed("Tensor '" + name + "' data_offsets must have exactly 2 elements, " - + "got " + offsets.length); + throw cursor.malformed("Tensor '" + name + "' data_offsets must have exactly 2 " + + "elements, got " + offsets.length); } dataOffsetBegin = offsets[0]; dataOffsetEnd = offsets[1]; } - default -> skipValue(); + default -> cursor.skipValue(); } - skipWhitespace(); - final char next = consume(); + cursor.skipWhitespace(); + final char next = cursor.consume(); if (next == ',') { - skipWhitespace(); + cursor.skipWhitespace(); continue; } if (next == '}') { if (dtype == null || shape == null || dataOffsetBegin < 0) { - throw malformed("Tensor '" + name + throw cursor.malformed("Tensor '" + name + "' is missing dtype, shape, or data_offsets"); } return new TensorInfo(name, dtype, shape, dataOffsetBegin, dataOffsetEnd); } - throw malformed("Expected ',' or '}' in tensor '" + name + "', got '" + next + "'"); + throw cursor.malformed("Expected ',' or '}' in tensor '" + name + "', got '" + next + "'"); } - throw malformed("Tensor '" + name + "' has an empty object; missing dtype, shape, " + throw cursor.malformed("Tensor '" + name + "' has an empty object; missing dtype, shape, " + "and data_offsets"); } private Map parseStringMap() { final Map map = new LinkedHashMap<>(); - expect('{'); - skipWhitespace(); - if (peek() == '}') { - position++; + cursor.expect('{'); + cursor.skipWhitespace(); + if (cursor.peek() == '}') { + cursor.consume(); return map; } while (true) { - skipWhitespace(); - final String key = parseString(); - skipWhitespace(); - expect(':'); - skipWhitespace(); - map.put(key, parseString()); - skipWhitespace(); - final char next = consume(); + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + map.put(key, cursor.parseString()); + cursor.skipWhitespace(); + final char next = cursor.consume(); if (next == ',') { continue; } if (next == '}') { return map; } - throw malformed("Expected ',' or '}' in __metadata__, got '" + next + "'"); + throw cursor.malformed("Expected ',' or '}' in __metadata__, got '" + next + "'"); } } @@ -181,7 +178,7 @@ private int[] parseIntArray() { final int[] ints = new int[longs.length]; for (int i = 0; i < longs.length; i++) { if (longs[i] < 0 || longs[i] > Integer.MAX_VALUE) { - throw malformed("Shape dimension out of int range: " + longs[i]); + throw cursor.malformed("Shape dimension out of int range: " + longs[i]); } ints[i] = (int) longs[i]; } @@ -189,25 +186,25 @@ private int[] parseIntArray() { } private long[] parseLongArray() { - expect('['); - skipWhitespace(); + cursor.expect('['); + cursor.skipWhitespace(); final List values = new ArrayList<>(); - if (peek() == ']') { - position++; + if (cursor.peek() == ']') { + cursor.consume(); return new long[0]; } while (true) { - skipWhitespace(); - values.add(parseLong()); - skipWhitespace(); - final char next = consume(); + cursor.skipWhitespace(); + values.add(cursor.parseLong()); + cursor.skipWhitespace(); + final char next = cursor.consume(); if (next == ',') { continue; } if (next == ']') { break; } - throw malformed("Expected ',' or ']' in a number array, got '" + next + "'"); + throw cursor.malformed("Expected ',' or ']' in a number array, got '" + next + "'"); } final long[] array = new long[values.size()]; for (int i = 0; i < array.length; i++) { @@ -216,180 +213,6 @@ private long[] parseLongArray() { return array; } - private long parseLong() { - final int start = position; - if (peek() == '-') { - position++; - } - if (position >= text.length() || !Character.isDigit(text.charAt(position))) { - throw malformed("Expected a non-negative integer"); - } - while (position < text.length() && Character.isDigit(text.charAt(position))) { - position++; - } - try { - return Long.parseLong(text.substring(start, position)); - } - catch (NumberFormatException e) { - throw malformed("Malformed integer: " + text.substring(start, position)); - } - } - - private String parseString() { - expect('"'); - final StringBuilder value = new StringBuilder(); - while (true) { - if (position >= text.length()) { - throw malformed("Unterminated string"); - } - final char c = text.charAt(position++); - if (c == '"') { - return value.toString(); - } - if (c == '\\') { - value.append(parseEscape()); - } - else { - value.append(c); - } - } - } - - private char parseEscape() { - if (position >= text.length()) { - throw malformed("Unterminated escape sequence"); - } - final char escape = text.charAt(position++); - return switch (escape) { - case '"' -> '"'; - case '\\' -> '\\'; - case '/' -> '/'; - case 'b' -> '\b'; - case 'f' -> '\f'; - case 'n' -> '\n'; - case 'r' -> '\r'; - case 't' -> '\t'; - case 'u' -> parseUnicodeEscape(); - default -> throw malformed("Unknown escape sequence: \\" + escape); - }; - } - - private char parseUnicodeEscape() { - if (position + 4 > text.length()) { - throw malformed("Truncated \\u escape sequence"); - } - final String hex = text.substring(position, position + 4); - position += 4; - try { - return (char) Integer.parseInt(hex, 16); - } - catch (NumberFormatException e) { - throw malformed("Malformed \\u escape sequence: " + hex); - } - } - - // Skips one JSON value of any type (string, number, array, object, true/false/null); used for - // header fields the reader does not care about (safetensors may add fields over time). - private void skipValue() { - skipWhitespace(); - final char c = peek(); - if (c == '"') { - parseString(); - } - else if (c == '[') { - position++; - skipWhitespace(); - if (peek() != ']') { - while (true) { - skipValue(); - skipWhitespace(); - final char next = consume(); - if (next == ',') { - skipWhitespace(); - continue; - } - if (next == ']') { - return; - } - throw malformed("Expected ',' or ']' while skipping an array, got '" + next + "'"); - } - } - position++; - } - else if (c == '{') { - position++; - skipWhitespace(); - if (peek() != '}') { - while (true) { - skipWhitespace(); - parseString(); - skipWhitespace(); - expect(':'); - skipValue(); - skipWhitespace(); - final char next = consume(); - if (next == ',') { - continue; - } - if (next == '}') { - return; - } - throw malformed("Expected ',' or '}' while skipping an object, got '" + next + "'"); - } - } - position++; - } - else if (c == '-' || Character.isDigit(c)) { - position++; - while (position < text.length() && "0123456789.eE+-".indexOf(text.charAt(position)) >= 0) { - position++; - } - } - else if (text.startsWith("true", position)) { - position += 4; - } - else if (text.startsWith("false", position)) { - position += 5; - } - else if (text.startsWith("null", position)) { - position += 4; - } - else { - throw malformed("Unexpected character while skipping a value: '" + c + "'"); - } - } - - private void skipWhitespace() { - while (position < text.length() && Character.isWhitespace(text.charAt(position))) { - position++; - } - } - - private char peek() { - if (position >= text.length()) { - throw malformed("Unexpected end of header"); - } - return text.charAt(position); - } - - private char consume() { - final char c = peek(); - position++; - return c; - } - - private void expect(char c) { - final char actual = consume(); - if (actual != c) { - throw malformed("Expected '" + c + "', got '" + actual + "'"); - } - } - - private IllegalArgumentException malformed(String message) { - return new IllegalArgumentException( - "Malformed safetensors header at offset " + position + ": " + message); - } - /** * The parsed header: the declared tensors, in header order, and the {@code __metadata__} * string map. diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index a6d417552f..c102445246 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -16,6 +16,7 @@ */ package opennlp.embeddings; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Set; @@ -56,6 +57,10 @@ public final class StaticEmbeddingModel { private static final float NORMALIZE_EPSILON = 1e-12f; private static final String WEIGHTS_TENSOR_NAME = "weights"; + private static final String VOCABULARY_FILE_NAME = "vocab.txt"; + private static final String SAFETENSORS_FILE_NAME = "model.safetensors"; + private static final String CONFIG_FILE_NAME = "config.json"; + private static final String TOKENIZER_CONFIG_FILE_NAME = "tokenizer_config.json"; private static final int[] NO_EXCLUDED_ROWS = new int[0]; // Never meaningful as a "similar word" result. private static final Set SPECIAL_TOKENS = Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, @@ -88,6 +93,78 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, this.specialRows = specialRows; } + /** + * Loads a static embedding model from a model directory, reading the tokenizer and pooling + * switches from the model's own configuration files instead of requiring the caller to know + * them: {@code normalize} from {@code config.json} and {@code do_lower_case} from + * {@code tokenizer_config.json}. The directory must contain {@code vocab.txt}, + * {@code model.safetensors}, {@code config.json}, and {@code tokenizer_config.json}, the + * layout Model2Vec-family releases publish (field names verified against published releases, + * not assumed). + * + *

A {@code strip_accents} that is absent or JSON {@code null} follows the BERT convention + * of stripping accents exactly when lower-casing, which is what the single lower-case switch + * of {@link #load(Path, Path, boolean, boolean)} does. A model that explicitly sets + * {@code strip_accents} against its {@code do_lower_case} value cannot be represented by + * that switch, so it is rejected rather than silently mis-tokenized.

+ * + * @param modelDirectory The model directory. Must not be {@code null} and must be a + * directory. + * @return The loaded model. + * @throws IllegalArgumentException Thrown if {@code modelDirectory} is {@code null} or not a + * directory, a required file is missing, a configuration file is malformed or lacks its + * field, the accent handling is not representable, or the vocabulary and the embedding + * matrix disagree; the message names the explicit overload as the fallback for + * differently laid-out models. + * @throws java.io.UncheckedIOException Thrown if reading a file fails. + */ + public static StaticEmbeddingModel load(Path modelDirectory) { + if (modelDirectory == null) { + throw new IllegalArgumentException("ModelDirectory must not be null"); + } + if (!Files.isDirectory(modelDirectory)) { + throw new IllegalArgumentException( + "Model directory does not exist or is not a directory: " + modelDirectory); + } + final Path vocabularyFile = requiredFile(modelDirectory, VOCABULARY_FILE_NAME); + final Path safetensorsFile = requiredFile(modelDirectory, SAFETENSORS_FILE_NAME); + final Path configFile = requiredFile(modelDirectory, CONFIG_FILE_NAME); + final Path tokenizerConfigFile = requiredFile(modelDirectory, TOKENIZER_CONFIG_FILE_NAME); + final Boolean normalize = FlatJsonFields.topLevelBoolean(configFile, "normalize"); + if (normalize == null) { + throw new IllegalArgumentException(configFile + " has no boolean 'normalize' field; " + + "use load(vocabularyFile, safetensorsFile, lowerCase, normalize) and choose " + + "explicitly"); + } + final Boolean lowerCase = + FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "do_lower_case"); + if (lowerCase == null) { + throw new IllegalArgumentException(tokenizerConfigFile + " has no boolean " + + "'do_lower_case' field; use load(vocabularyFile, safetensorsFile, lowerCase, " + + "normalize) and choose explicitly"); + } + final Boolean stripAccents = + FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "strip_accents"); + if (stripAccents != null && !stripAccents.equals(lowerCase)) { + throw new IllegalArgumentException(tokenizerConfigFile + " sets strip_accents=" + + stripAccents + " against do_lower_case=" + lowerCase + "; the single lower-case " + + "switch strips accents exactly when lower-casing, so this model must be loaded " + + "with load(vocabularyFile, safetensorsFile, lowerCase, normalize) after choosing " + + "deliberately"); + } + return load(vocabularyFile, safetensorsFile, lowerCase, normalize); + } + + private static Path requiredFile(Path modelDirectory, String name) { + final Path file = modelDirectory.resolve(name); + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("Model directory " + modelDirectory + " has no " + + name + "; for a different layout, use load(vocabularyFile, safetensorsFile, " + + "lowerCase, normalize)"); + } + return file; + } + /** * Loads a static embedding model from a BERT-style {@code vocab.txt} and a safetensors weight * file, the file pair a Model2Vec-family distillation publishes. No model is bundled with this diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java new file mode 100644 index 0000000000..27fee69549 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FlatJsonFieldsTest { + + private static Path write(Path dir, String json) throws IOException { + final Path file = dir.resolve("config.json"); + Files.writeString(file, json); + return file; + } + + @Test + void testReadsTopLevelBooleans(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"normalize\":true,\"do_lower_case\":false}"); + + assertEquals(Boolean.TRUE, FlatJsonFields.topLevelBoolean(file, "normalize")); + assertEquals(Boolean.FALSE, FlatJsonFields.topLevelBoolean(file, "do_lower_case")); + } + + @Test + void testAbsentFieldAndExplicitNullBothReadAsNull(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"strip_accents\":null}"); + + assertNull(FlatJsonFields.topLevelBoolean(file, "strip_accents")); + assertNull(FlatJsonFields.topLevelBoolean(file, "missing")); + } + + @Test + void testSkipsFieldsOfEveryOtherType(@TempDir Path dir) throws IOException { + // The shapes real tokenizer_config.json files carry around the looked-up field: nested + // objects, arrays, floats, and strings must all be skipped structurally. + final Path file = write(dir, "{\"added_tokens_decoder\":{\"0\":{\"special\":true}}," + + "\"model_max_length\":1.0E9,\"architectures\":[\"StaticModel\"]," + + "\"cls_token\":\"[CLS]\",\"normalize\":true}"); + + assertEquals(Boolean.TRUE, FlatJsonFields.topLevelBoolean(file, "normalize")); + } + + @Test + void testNestedOccurrencesOfTheNameDoNotMatch(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"outer\":{\"normalize\":true}}"); + + assertNull(FlatJsonFields.topLevelBoolean(file, "normalize")); + } + + @Test + void testToleratesAnEmptyObjectAndTrailingWhitespace(@TempDir Path dir) throws IOException { + assertNull(FlatJsonFields.topLevelBoolean(write(dir, "{} \n"), "normalize")); + } + + @Test + void testRejectsANonBooleanValue(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"normalize\":\"yes\"}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelBoolean(file, "normalize")); + assertTrue(e.getMessage().contains("must be a boolean")); + } + + @Test + void testRejectsADuplicateField(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"normalize\":true,\"normalize\":false}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelBoolean(file, "normalize")); + assertTrue(e.getMessage().contains("more than once")); + } + + @Test + void testRejectsMalformedJsonWithTheFileNameInTheMessage(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"normalize\" true}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelBoolean(file, "normalize")); + assertTrue(e.getMessage().contains("config.json")); + } + + @Test + void testRejectsTrailingGarbage(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{} x"); + + assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelBoolean(file, "normalize")); + } + + @Test + void testMissingFileFailsAsAnIoProblem(@TempDir Path dir) { + assertThrows(UncheckedIOException.class, + () -> FlatJsonFields.topLevelBoolean(dir.resolve("absent.json"), "normalize")); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index 1424472843..31c7becdfd 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -238,4 +238,90 @@ void testLoadRejectsWeightsSizeMismatch(@TempDir Path dir) throws IOException { () -> StaticEmbeddingModel.load(writeVocab(dir), file, true, false)); assertTrue(e.getMessage().contains("weights")); } + + // Writes the two JSON configuration files of a published model directory alongside the + // vocab/safetensors fixtures, with the shapes real releases use (extra fields, floats, + // nested objects, an explicit strip_accents null). + private static void writeConfigs(Path dir, String normalize, String doLowerCase) + throws IOException { + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"architectures\":[\"StaticModel\"]," + + "\"apply_pca\":256,\"normalize\":" + normalize + ",\"hidden_dim\":3}"); + Files.writeString(dir.resolve("tokenizer_config.json"), + "{\"added_tokens_decoder\":{\"0\":{\"content\":\"[PAD]\",\"special\":true}}," + + "\"do_lower_case\":" + doLowerCase + ",\"strip_accents\":null," + + "\"tokenizer_class\":\"BertTokenizer\"}"); + } + + @Test + void testLoadsFromAModelDirectory(@TempDir Path dir) throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + writeConfigs(dir, "false", "true"); + + final StaticEmbeddingModel model = StaticEmbeddingModel.load(dir); + + // Same fixture and switches as testEmbedMeanPoolsWithoutWeights, resolved from the configs + // this time; the upper-cased input additionally proves do_lower_case was picked up. + assertArrayEquals(new float[] {3.5f, 35f, 350f}, model.embed("HELLO WORLD"), 1e-5f); + } + + @Test + void testDirectoryLoadReadsNormalizeFromTheConfig(@TempDir Path dir) throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + writeConfigs(dir, "true", "true"); + + final float[] result = StaticEmbeddingModel.load(dir).embed("cat"); + + double normSquared = 0; + for (final float v : result) { + normSquared += (double) v * v; + } + assertEquals(1.0, Math.sqrt(normSquared), 1e-5); + } + + @Test + void testDirectoryLoadRejectsNullAndNonDirectory(@TempDir Path dir) { + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(null)); + assertThrows(IllegalArgumentException.class, + () -> StaticEmbeddingModel.load(dir.resolve("absent"))); + } + + @Test + void testDirectoryLoadNamesTheMissingFile(@TempDir Path dir) throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + // no config.json, no tokenizer_config.json + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + assertTrue(e.getMessage().contains("config.json")); + assertTrue(e.getMessage().contains("load(vocabularyFile, safetensorsFile")); + } + + @Test + void testDirectoryLoadRejectsAConfigWithoutNormalize(@TempDir Path dir) throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + writeConfigs(dir, "false", "true"); + Files.writeString(dir.resolve("config.json"), "{\"model_type\":\"model2vec\"}"); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + assertTrue(e.getMessage().contains("normalize")); + } + + @Test + void testDirectoryLoadRejectsContradictoryStripAccents(@TempDir Path dir) throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + writeConfigs(dir, "false", "true"); + Files.writeString(dir.resolve("tokenizer_config.json"), + "{\"do_lower_case\":true,\"strip_accents\":false}"); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + assertTrue(e.getMessage().contains("strip_accents")); + } } From e25ff872c8f73fa29955ef7c8522c3fccc82dc68 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 9 Jul 2026 16:27:29 -0400 Subject: [PATCH 32/82] Document opennlp-embeddings: Dev Manual chapter and module README Covers when to use a static table over a contextual model, the one-call model-directory load and the explicit overload, thread safety, the no-bundled-model license posture, and the safetensors reader's guarantees. --- opennlp-docs/src/docbkx/embeddings.xml | 96 +++++++++++++++++++ opennlp-docs/src/docbkx/opennlp.xml | 1 + .../opennlp-embeddings/README.md | 54 +++++++++++ 3 files changed, 151 insertions(+) create mode 100644 opennlp-docs/src/docbkx/embeddings.xml create mode 100644 opennlp-extensions/opennlp-embeddings/README.md diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml new file mode 100644 index 0000000000..09e2059aad --- /dev/null +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -0,0 +1,96 @@ + + + + + + + Static Embeddings + +
+ Introduction + + The opennlp-embeddings extension module produces sentence and word + embedding vectors from a static (non-contextual) embedding table: a per-token vector + matrix plus WordPiece tokenization. It is the modern successor to the word2vec and + GloVe workflow. Distillation tools can compress a sentence-transformer into such a + flat table (the Model2Vec family of releases is the primary target), and looking a + sentence up in the table approximates the transformer's semantics at a small fraction + of the cost: embedding a text is tokenize, gather, mean-pool, and normalize, with no + model forward pass, no GPU, and no native runtime. + + + Use it when embedding throughput and a pure-JVM deployment matter more than the last + few points of retrieval quality: semantic similarity and deduplication, candidate + retrieval for a heavier reranker, clustering, or classification features. A + contextual model remains the better choice when distinguishing word senses in context + is the point of the task. + + + No model is bundled with the module. Callers point it at a model directory they + downloaded; the table's own license applies to the table. + +
+ +
+ Embedding Text with the API + + A model directory containing vocab.txt, model.safetensors, + config.json, and tokenizer_config.json (the layout published + model releases use) loads with a single call; the tokenizer and pooling switches are + read from the model's own configuration files: + + + neighbors = model.mostSimilar("coffee", 5); +List analogy = model.analogy("man", "king", "woman", 1);]]> + + + For a model laid out differently, the explicit overload takes the two data files and + the two switches directly: whether the tokenizer lower-cases (and strips accents), + and whether embeddings are L2-normalized. Both are properties of the model, published + in its configuration. + + + + + + Instances are immutable and safe for concurrent use, so one loaded model can serve + every thread of an application. Texts with no in-vocabulary tokens embed to a zero + vector rather than raising an error, and similarity reports + 0 for them. + +
+ +
+ The safetensors Reader + + Weights are read with a purpose-built reader for the safetensors format. Unlike + pickle-based checkpoint formats, a safetensors file carries no executable content + (the header is data-only JSON and the body is raw tensor bytes), so loading one + cannot execute arbitrary code. Only the header is read eagerly; tensor data streams + directly into the decoded array, so the file size is not limited by Java's + int-indexed arrays. One decoded tensor is capped at the maximum Java array length + (about 2.1 billion float elements), checked explicitly. + +
+ diff --git a/opennlp-docs/src/docbkx/opennlp.xml b/opennlp-docs/src/docbkx/opennlp.xml index 36641c2c89..7d78e290bd 100644 --- a/opennlp-docs/src/docbkx/opennlp.xml +++ b/opennlp-docs/src/docbkx/opennlp.xml @@ -120,6 +120,7 @@ under the License. + diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md new file mode 100644 index 0000000000..cc924cb49c --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -0,0 +1,54 @@ + + +# OpenNLP Static Embeddings + +This module produces sentence and word embedding vectors from a static (non-contextual) embedding table: a per-token vector matrix plus WordPiece tokenization, the modern successor to the word2vec and GloVe workflow. Distillation tools can compress a sentence-transformer into such a flat table (the Model2Vec family of releases is the primary target), and looking a sentence up in the table approximates the transformer's semantics at a small fraction of the cost: embedding a text is tokenize, gather, mean-pool, and normalize. No model forward pass, no GPU, no native runtime, pure JVM. + +## When to use it + +Use this module when embedding throughput and deployment simplicity matter more than the last few points of retrieval quality: semantic similarity and deduplication, candidate retrieval for a heavier reranker, clustering, or classification features. A contextual model remains the better choice when distinguishing word senses in context is the point of the task. + +## Usage + +A downloaded model directory containing `vocab.txt`, `model.safetensors`, `config.json`, and `tokenizer_config.json` (the layout published releases use) loads with one call; the tokenizer and pooling switches are read from the model's own configuration: + +```java +StaticEmbeddingModel model = StaticEmbeddingModel.load(Path.of("/path/to/model-directory")); + +float[] vector = model.embed("The quick brown fox"); +double similarity = model.similarity("coffee", "espresso"); +List neighbors = model.mostSimilar("coffee", 5); +List analogy = model.analogy("man", "king", "woman", 1); +``` + +For a model laid out differently, the explicit overload takes the two data files and the two model properties directly: + +```java +StaticEmbeddingModel model = StaticEmbeddingModel.load( + Path.of("vocab.txt"), Path.of("model.safetensors"), + true, // lowerCase, from the model's do_lower_case + true); // normalize, from the model's config +``` + +Instances are immutable and safe for concurrent use, so one loaded model can serve every thread of an application. Texts with no in-vocabulary tokens embed to a zero vector rather than raising an error. + +## Notes + +- No model is bundled. Callers point the module at files they downloaded, and the table's own license applies to the table. +- Weights are read with a purpose-built safetensors reader. Unlike pickle-based checkpoint formats, safetensors carries no executable content, so loading a file cannot execute arbitrary code. Tensor data streams directly into the decoded array, so file size is not limited by Java's int-indexed arrays; one decoded tensor is capped at the maximum Java array length (about 2.1 billion float elements), checked explicitly. +- The pooling formula matches the reference implementations of the targeted model family exactly (verified against them, not assumed): special tokens never pool, unknown tokens are dropped, per-token weights multiply into the sum, and the sum divides by the plain token count. From c7610dee0c17c9425dd95cf90947253f211c2c85 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 10 Jul 2026 16:49:27 -0400 Subject: [PATCH 33/82] OPENNLP-1877: Address review: value-safe TensorInfo, checked IO, typed load switches, distribution wiring TensorInfo copies its shape on construction and access and compares by value, so callers cannot corrupt validated metadata and equality behaves like exported API should; elementCount() detects overflow from crafted headers. The load and read entry points now declare IOException like the rest of the project's resource loaders instead of wrapping in UncheckedIOException. The two adjacent boolean load flags became the Casing and Normalization enums, so the switches cannot be swapped silently; the directory-based load maps the model's configuration onto them and the manual, README, tests, and benchmark use the new signature. The binary distribution now ships the module: opennlp-embeddings added to the distr dependencies and its apidocs to the assembly, matching the other extension modules. JSON parsing tightened to the fail-loud contract: unicode escapes must be four hex digits (Integer.parseInt also accepted signed input) and numbers in skipped fields are held to the JSON grammar. WordPieceVocabulary gains its own test class covering the line-number ids, the duplicate rejection, the -1 sentinel, and reverse-lookup bounds, which now fail loud instead of leaking an index exception. The safetensors fixture writer that was copied across the tests is now a shared test utility, the stale cross-module javadoc reference is gone, the element-count versus byte-range guard has its negative test, and else/catch placement matches the project style. --- opennlp-distr/pom.xml | 5 ++ opennlp-distr/src/main/assembly/bin.xml | 7 ++ opennlp-docs/src/docbkx/embeddings.xml | 4 +- .../opennlp-embeddings/README.md | 4 +- .../StaticEmbeddingModelBenchmark.java | 5 +- .../opennlp/embeddings/FlatJsonFields.java | 25 ++---- .../java/opennlp/embeddings/JsonCursor.java | 76 +++++++++++----- .../java/opennlp/embeddings/Neighbor.java | 1 + .../opennlp/embeddings/SafetensorsFile.java | 15 +--- .../embeddings/SafetensorsHeaderParser.java | 3 +- .../embeddings/StaticEmbeddingModel.java | 71 +++++++++++---- .../java/opennlp/embeddings/TensorInfo.java | 53 ++++++++++- .../embeddings/WordPieceVocabulary.java | 18 ++-- .../embeddings/FlatJsonFieldsTest.java | 4 +- .../embeddings/SafetensorsFileTest.java | 42 +++++++++ .../SafetensorsHeaderParserTest.java | 22 +++++ .../embeddings/SafetensorsTestFiles.java | 88 +++++++++++++++++++ .../StaticEmbeddingModelConcurrencyTest.java | 32 ++----- .../StaticEmbeddingModelSimilarityTest.java | 27 ++---- .../embeddings/StaticEmbeddingModelTest.java | 76 +++++++--------- .../embeddings/WordPieceVocabularyTest.java | 81 +++++++++++++++++ 21 files changed, 479 insertions(+), 180 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordPieceVocabularyTest.java diff --git a/opennlp-distr/pom.xml b/opennlp-distr/pom.xml index 7e8501bca9..c1cf8e841c 100644 --- a/opennlp-distr/pom.xml +++ b/opennlp-distr/pom.xml @@ -96,6 +96,11 @@ opennlp-subword + + org.apache.opennlp + opennlp-embeddings + + org.apache.opennlp diff --git a/opennlp-distr/src/main/assembly/bin.xml b/opennlp-distr/src/main/assembly/bin.xml index b861de7caf..0981003467 100644 --- a/opennlp-distr/src/main/assembly/bin.xml +++ b/opennlp-distr/src/main/assembly/bin.xml @@ -232,6 +232,13 @@ docs/apidocs/opennlp-morfologik + + ../opennlp-extensions/opennlp-embeddings/target/reports/apidocs + 644 + 755 + docs/apidocs/opennlp-embeddings + + ../opennlp-extensions/opennlp-spellcheck/target/reports/apidocs 644 diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 09e2059aad..abfe59c075 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -69,8 +69,8 @@ List analogy = model.analogy("man", "king", "woman", 1);]]> diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md index cc924cb49c..5c64565f80 100644 --- a/opennlp-extensions/opennlp-embeddings/README.md +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -41,8 +41,8 @@ For a model laid out differently, the explicit overload takes the two data files ```java StaticEmbeddingModel model = StaticEmbeddingModel.load( Path.of("vocab.txt"), Path.of("model.safetensors"), - true, // lowerCase, from the model's do_lower_case - true); // normalize, from the model's config + StaticEmbeddingModel.Casing.UNCASED, // from the model's do_lower_case + StaticEmbeddingModel.Normalization.L2); // from the model's config ``` Instances are immutable and safe for concurrent use, so one loaded model can serve every thread of an application. Texts with no in-vocabulary tokens embed to a zero vector rather than raising an error. diff --git a/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java index 03b5d75717..1ede49337b 100644 --- a/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java +++ b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java @@ -45,6 +45,8 @@ import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; +import opennlp.embeddings.StaticEmbeddingModel.Casing; +import opennlp.embeddings.StaticEmbeddingModel.Normalization; /** * JMH benchmark for {@link StaticEmbeddingModel}, the raw-lookup-throughput number the module's @@ -92,7 +94,8 @@ public void load() throws IOException { tempDir = Files.createTempDirectory("opennlp-embeddings-jmh"); final Path vocabFile = writeVocab(tempDir); final Path safetensorsFile = writeSafetensors(tempDir); - model = StaticEmbeddingModel.load(vocabFile, safetensorsFile, true, true); + model = StaticEmbeddingModel.load(vocabFile, safetensorsFile, + Casing.UNCASED, Normalization.L2); } @TearDown(Level.Trial) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java index 6ba963dbd4..81fb969676 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java @@ -17,7 +17,6 @@ package opennlp.embeddings; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; @@ -44,16 +43,10 @@ private FlatJsonFields() { * {@code null} (the formats treat those the same: fall back to the default). * @throws IllegalArgumentException Thrown if the file is not a well-formed JSON object, the * field appears more than once, or its value is neither a boolean nor {@code null}. - * @throws UncheckedIOException Thrown if reading the file fails. + * @throws IOException Thrown if reading the file fails. */ - static Boolean topLevelBoolean(Path file, String field) { - final String json; - try { - json = Files.readString(file); - } - catch (IOException e) { - throw new UncheckedIOException("Unable to read " + file, e); - } + static Boolean topLevelBoolean(Path file, String field) throws IOException { + final String json = Files.readString(file); final JsonCursor cursor = new JsonCursor(json, file.getFileName().toString()); cursor.skipWhitespace(); cursor.expect('{'); @@ -62,8 +55,7 @@ static Boolean topLevelBoolean(Path file, String field) { boolean seen = false; if (cursor.peek() == '}') { cursor.consume(); - } - else { + } else { while (true) { cursor.skipWhitespace(); final String key = cursor.parseString(); @@ -77,15 +69,12 @@ static Boolean topLevelBoolean(Path file, String field) { seen = true; if (cursor.consumeLiteral("true")) { value = Boolean.TRUE; - } - else if (cursor.consumeLiteral("false")) { + } else if (cursor.consumeLiteral("false")) { value = Boolean.FALSE; - } - else if (!cursor.consumeLiteral("null")) { + } else if (!cursor.consumeLiteral("null")) { throw cursor.malformed("Field '" + field + "' must be a boolean or null"); } - } - else { + } else { cursor.skipValue(); } cursor.skipWhitespace(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java index 6947950d0a..237b6e411b 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java @@ -96,8 +96,7 @@ String parseString() { } if (c == '\\') { value.append(parseEscape()); - } - else { + } else { value.append(c); } } @@ -128,11 +127,53 @@ private char parseUnicodeEscape() { } final String hex = text.substring(position, position + 4); position += 4; - try { - return (char) Integer.parseInt(hex, 16); + // Each of the four characters must be a hex digit; Integer.parseInt alone would also + // accept a sign and silently decode the wrong character. + int value = 0; + for (int i = 0; i < 4; i++) { + final int digit = Character.digit(hex.charAt(i), 16); + if (digit < 0) { + throw malformed("Malformed \\u escape sequence: " + hex); + } + value = (value << 4) | digit; } - catch (NumberFormatException e) { - throw malformed("Malformed \\u escape sequence: " + hex); + return (char) value; + } + + // Skips one number, holding it to the JSON grammar (optional minus, digits, optional + // fraction, optional signed exponent) so malformed input fails loud even in skipped fields. + private void skipNumber() { + if (peek() == '-') { + position++; + } + if (position >= text.length() || !Character.isDigit(text.charAt(position))) { + throw malformed("Malformed number"); + } + while (position < text.length() && Character.isDigit(text.charAt(position))) { + position++; + } + if (position < text.length() && text.charAt(position) == '.') { + position++; + if (position >= text.length() || !Character.isDigit(text.charAt(position))) { + throw malformed("Malformed number: digit expected after the decimal point"); + } + while (position < text.length() && Character.isDigit(text.charAt(position))) { + position++; + } + } + if (position < text.length() + && (text.charAt(position) == 'e' || text.charAt(position) == 'E')) { + position++; + if (position < text.length() + && (text.charAt(position) == '+' || text.charAt(position) == '-')) { + position++; + } + if (position >= text.length() || !Character.isDigit(text.charAt(position))) { + throw malformed("Malformed number: digit expected in the exponent"); + } + while (position < text.length() && Character.isDigit(text.charAt(position))) { + position++; + } } } @@ -149,8 +190,7 @@ long parseLong() { } try { return Long.parseLong(text.substring(start, position)); - } - catch (NumberFormatException e) { + } catch (NumberFormatException e) { throw malformed("Malformed integer: " + text.substring(start, position)); } } @@ -162,8 +202,7 @@ void skipValue() { final char c = peek(); if (c == '"') { parseString(); - } - else if (c == '[') { + } else if (c == '[') { position++; skipWhitespace(); if (peek() != ']') { @@ -182,8 +221,7 @@ else if (c == '[') { } } position++; - } - else if (c == '{') { + } else if (c == '{') { position++; skipWhitespace(); if (peek() != '}') { @@ -205,17 +243,11 @@ else if (c == '{') { } } position++; - } - else if (c == '-' || Character.isDigit(c)) { - position++; - while (position < text.length() && "0123456789.eE+-".indexOf(text.charAt(position)) >= 0) { - position++; - } - } - else if (consumeLiteral("true") || consumeLiteral("false") || consumeLiteral("null")) { + } else if (c == '-' || Character.isDigit(c)) { + skipNumber(); + } else if (consumeLiteral("true") || consumeLiteral("false") || consumeLiteral("null")) { // consumed, nothing to record - } - else { + } else { throw malformed("Unexpected character while skipping a value: '" + c + "'"); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java index 68d6f3d58e..b37a9b7808 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java @@ -16,6 +16,7 @@ */ package opennlp.embeddings; + /** * One vocabulary token found near a query vector by {@link StaticEmbeddingModel#mostSimilar} * or {@link StaticEmbeddingModel#analogy}, most similar first. diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 3fd36d2b0a..36c048b160 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -17,7 +17,6 @@ package opennlp.embeddings; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; @@ -89,9 +88,9 @@ private SafetensorsFile(Path file, long dataStart, Map tenso * file's actual length. * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing, or the * file is malformed. - * @throws UncheckedIOException Thrown if reading the file fails. + * @throws IOException Thrown if reading the file fails. */ - public static SafetensorsFile read(Path file) { + public static SafetensorsFile read(Path file) throws IOException { if (file == null) { throw new IllegalArgumentException("File must not be null"); } @@ -139,9 +138,6 @@ public static SafetensorsFile read(Path file) { return new SafetensorsFile(file, dataStart, Collections.unmodifiableMap(tensorsByName), Collections.unmodifiableMap(parsed.metadata())); } - catch (IOException e) { - throw new UncheckedIOException("Unable to read safetensors file " + file, e); - } } /** {@return the names of every tensor declared in the header, in header order} */ @@ -178,9 +174,9 @@ public TensorInfo tensorInfo(String name) { * this file, not declared with dtype {@code F32}, or larger than a Java array can hold. * @throws IllegalStateException Thrown if the file has been truncated since * {@link #read(Path)} validated the tensor's byte range. - * @throws UncheckedIOException Thrown if reading the file fails. + * @throws IOException Thrown if reading the file fails. */ - public float[] readFloat32(String name) { + public float[] readFloat32(String name) throws IOException { final TensorInfo info = tensorInfo(name); if (!"F32".equals(info.dtype())) { throw new IllegalArgumentException( @@ -222,9 +218,6 @@ public float[] readFloat32(String name) { } return values; } - catch (IOException e) { - throw new UncheckedIOException("Unable to read tensor '" + name + "' from " + file, e); - } } // Fills the buffer with bytes starting at the given file position; fails loud if the file diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java index 8a6b0989e3..41a54cb905 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java @@ -76,8 +76,7 @@ private Result parseTop() { cursor.skipWhitespace(); if (METADATA_KEY.equals(key)) { metadata = parseStringMap(); - } - else { + } else { tensors.add(parseTensorInfo(key)); } cursor.skipWhitespace(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index c102445246..99755c8fe8 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -16,6 +16,7 @@ */ package opennlp.embeddings; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -55,6 +56,26 @@ @ThreadSafe public final class StaticEmbeddingModel { + /** How the tokenizer treats letter case, matching the base model's tokenizer configuration. */ + public enum Casing { + + /** Lower-case and strip accents, the uncased BGE/BERT convention. */ + UNCASED, + + /** Preserve case and accents. */ + CASED + } + + /** Whether pooled vectors are length-normalized, matching the model's configuration. */ + public enum Normalization { + + /** L2-normalize each pooled vector. */ + L2, + + /** Leave pooled vectors unnormalized. */ + NONE + } + private static final float NORMALIZE_EPSILON = 1e-12f; private static final String WEIGHTS_TENSOR_NAME = "weights"; private static final String VOCABULARY_FILE_NAME = "vocab.txt"; @@ -104,7 +125,7 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, * *

A {@code strip_accents} that is absent or JSON {@code null} follows the BERT convention * of stripping accents exactly when lower-casing, which is what the single lower-case switch - * of {@link #load(Path, Path, boolean, boolean)} does. A model that explicitly sets + * of {@link #load(Path, Path, Casing, Normalization)} does. A model that explicitly sets * {@code strip_accents} against its {@code do_lower_case} value cannot be represented by * that switch, so it is rejected rather than silently mis-tokenized.

* @@ -116,9 +137,9 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, * field, the accent handling is not representable, or the vocabulary and the embedding * matrix disagree; the message names the explicit overload as the fallback for * differently laid-out models. - * @throws java.io.UncheckedIOException Thrown if reading a file fails. + * @throws IOException Thrown if reading a file fails. */ - public static StaticEmbeddingModel load(Path modelDirectory) { + public static StaticEmbeddingModel load(Path modelDirectory) throws IOException { if (modelDirectory == null) { throw new IllegalArgumentException("ModelDirectory must not be null"); } @@ -133,15 +154,15 @@ public static StaticEmbeddingModel load(Path modelDirectory) { final Boolean normalize = FlatJsonFields.topLevelBoolean(configFile, "normalize"); if (normalize == null) { throw new IllegalArgumentException(configFile + " has no boolean 'normalize' field; " - + "use load(vocabularyFile, safetensorsFile, lowerCase, normalize) and choose " + + "use load(vocabularyFile, safetensorsFile, casing, normalization) and choose " + "explicitly"); } final Boolean lowerCase = FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "do_lower_case"); if (lowerCase == null) { throw new IllegalArgumentException(tokenizerConfigFile + " has no boolean " - + "'do_lower_case' field; use load(vocabularyFile, safetensorsFile, lowerCase, " - + "normalize) and choose explicitly"); + + "'do_lower_case' field; use load(vocabularyFile, safetensorsFile, casing, " + + "normalization) and choose explicitly"); } final Boolean stripAccents = FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "strip_accents"); @@ -149,10 +170,12 @@ public static StaticEmbeddingModel load(Path modelDirectory) { throw new IllegalArgumentException(tokenizerConfigFile + " sets strip_accents=" + stripAccents + " against do_lower_case=" + lowerCase + "; the single lower-case " + "switch strips accents exactly when lower-casing, so this model must be loaded " - + "with load(vocabularyFile, safetensorsFile, lowerCase, normalize) after choosing " + + "with load(vocabularyFile, safetensorsFile, casing, normalization) after choosing " + "deliberately"); } - return load(vocabularyFile, safetensorsFile, lowerCase, normalize); + return load(vocabularyFile, safetensorsFile, + lowerCase ? Casing.UNCASED : Casing.CASED, + normalize ? Normalization.L2 : Normalization.NONE); } private static Path requiredFile(Path modelDirectory, String name) { @@ -160,7 +183,7 @@ private static Path requiredFile(Path modelDirectory, String name) { if (!Files.isRegularFile(file)) { throw new IllegalArgumentException("Model directory " + modelDirectory + " has no " + name + "; for a different layout, use load(vocabularyFile, safetensorsFile, " - + "lowerCase, normalize)"); + + "casing, normalization)"); } return file; } @@ -179,23 +202,35 @@ private static Path requiredFile(Path modelDirectory, String name) { * An optional 1-D {@code F32} tensor named {@code "weights"}, one * scalar per vocabulary row, is used as a per-token pooling weight * when present. - * @param lowerCase Whether the tokenizer should lower-case and strip accents, matching - * the base model's tokenizer configuration ({@code true} for the - * uncased BGE/BERT family this module targets). - * @param normalize Whether {@link #embed(String)} L2-normalizes its result, matching - * the source model's {@code config.json} {@code normalize} field. + * @param casing Whether the tokenizer lower-cases and strips accents + * ({@link Casing#UNCASED}, matching the uncased BGE/BERT family this + * module targets) or preserves case ({@link Casing#CASED}), matching + * the base model's tokenizer configuration. + * @param normalization Whether {@link #embed(String)} L2-normalizes its result + * ({@link Normalization#L2}), matching the source model's + * {@code config.json} {@code normalize} field. * @return The loaded model. * @throws IllegalArgumentException Thrown if an argument is {@code null}, a file is missing * or malformed, or the vocabulary size and the embedding matrix's row count disagree. + * @throws IOException Thrown if reading a file fails. */ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFile, - boolean lowerCase, boolean normalize) { + Casing casing, Normalization normalization) + throws IOException { if (vocabularyFile == null) { throw new IllegalArgumentException("VocabularyFile must not be null"); } if (safetensorsFile == null) { throw new IllegalArgumentException("SafetensorsFile must not be null"); } + if (casing == null) { + throw new IllegalArgumentException("Casing must not be null"); + } + if (normalization == null) { + throw new IllegalArgumentException("Normalization must not be null"); + } + final boolean lowerCase = casing == Casing.UNCASED; + final boolean normalize = normalization == Normalization.L2; final WordPieceVocabulary vocabulary = WordPieceVocabulary.read(vocabularyFile); final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); @@ -276,8 +311,7 @@ public float[] embed(String text) { for (int d = 0; d < dimension; d++) { sum[d] += embeddings[base + d]; } - } - else { + } else { final float weight = weights[row]; for (int d = 0; d < dimension; d++) { sum[d] += embeddings[base + d] * weight; @@ -529,8 +563,7 @@ void offer(int row, double similarity) { swap(parent, i); i = parent; } - } - else if (similarity > similarities[0]) { + } else if (similarity > similarities[0]) { similarities[0] = similarity; rows[0] = row; siftDown(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java index 64c0bb8d89..e07c7c212b 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java @@ -16,6 +16,9 @@ */ package opennlp.embeddings; +import java.util.Arrays; + + /** * Header metadata for one tensor in a safetensors file, as declared by the file's own JSON * header. Carries no data; {@link SafetensorsFile#readFloat32(String)} resolves the bytes. @@ -32,14 +35,62 @@ public record TensorInfo(String name, String dtype, int[] shape, long dataOffsetBegin, long dataOffsetEnd) { + /** + * Creates the metadata, copying {@code shape} so later mutation of the caller's array cannot + * corrupt the validated state. + */ + public TensorInfo { + shape = shape.clone(); + } + + /** + * @return The tensor's dimensions, outermost first, as a copy; mutating it does not affect + * this record. + */ + @Override + public int[] shape() { + return shape.clone(); + } + /** * @return The number of elements the tensor holds, the product of {@link #shape()}. + * @throws IllegalArgumentException Thrown if the product overflows a {@code long}, which only + * a crafted header can produce. */ public long elementCount() { long count = 1; for (int dimension : shape) { - count *= dimension; + try { + count = Math.multiplyExact(count, dimension); + } catch (ArithmeticException e) { + throw new IllegalArgumentException("Tensor '" + name + "' declares a shape " + + Arrays.toString(shape) + " whose element count overflows a long", e); + } } return count; } + + @Override + public boolean equals(Object other) { + return other instanceof TensorInfo that + && name.equals(that.name) && dtype.equals(that.dtype) + && Arrays.equals(shape, that.shape) + && dataOffsetBegin == that.dataOffsetBegin && dataOffsetEnd == that.dataOffsetEnd; + } + + @Override + public int hashCode() { + int result = name.hashCode(); + result = 31 * result + dtype.hashCode(); + result = 31 * result + Arrays.hashCode(shape); + result = 31 * result + Long.hashCode(dataOffsetBegin); + result = 31 * result + Long.hashCode(dataOffsetEnd); + return result; + } + + @Override + public String toString() { + return "TensorInfo[name=" + name + ", dtype=" + dtype + ", shape=" + Arrays.toString(shape) + + ", dataOffsetBegin=" + dataOffsetBegin + ", dataOffsetEnd=" + dataOffsetEnd + "]"; + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java index cdff03c833..7cfd65d10f 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java @@ -17,7 +17,6 @@ package opennlp.embeddings; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; @@ -55,23 +54,16 @@ private WordPieceVocabulary(Map idByToken, List tokenBy * @return The parsed vocabulary. * @throws IllegalArgumentException Thrown if {@code file} is {@code null}, missing, or * contains a duplicate token. - * @throws UncheckedIOException Thrown if reading the file fails. + * @throws IOException Thrown if reading the file fails. */ - static WordPieceVocabulary read(Path file) { + static WordPieceVocabulary read(Path file) throws IOException { if (file == null) { throw new IllegalArgumentException("File must not be null"); } if (!Files.isRegularFile(file)) { throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); } - final List lines; - try { - lines = Files.readAllLines(file); - } - catch (IOException e) { - throw new UncheckedIOException("Unable to read vocabulary file " + file, e); - } - return fromLines(lines, file.toString()); + return fromLines(Files.readAllLines(file), file.toString()); } // Package-private so tests can build a vocabulary from in-memory lines without a temp file. @@ -121,6 +113,10 @@ int size() { * @return The token at that id. */ String token(int id) { + if (id < 0 || id >= tokenById.size()) { + throw new IllegalArgumentException( + "Id " + id + " is outside [0, " + tokenById.size() + ")"); + } return tokenById.get(id); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java index 27fee69549..c37bb51ad5 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java @@ -17,7 +17,7 @@ package opennlp.embeddings; import java.io.IOException; -import java.io.UncheckedIOException; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -113,7 +113,7 @@ void testRejectsTrailingGarbage(@TempDir Path dir) throws IOException { @Test void testMissingFileFailsAsAnIoProblem(@TempDir Path dir) { - assertThrows(UncheckedIOException.class, + assertThrows(IOException.class, () -> FlatJsonFields.topLevelBoolean(dir.resolve("absent.json"), "normalize")); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java index 21a6cb30d6..1595fd2c26 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java @@ -291,4 +291,46 @@ void testFailsLoudWhenTheFileIsTruncatedAfterRead(@TempDir Path dir) throws IOEx assertThrows(IllegalStateException.class, () -> parsed.readFloat32("w")); assertTrue(e.getMessage().contains("truncated")); } + + @Test + void testReadFloat32RejectsElementCountByteRangeMismatch(@TempDir Path dir) throws IOException { + // Shape [2] declares two F32 elements (8 bytes) but the data range holds only one. + final byte[] data = floatsToLittleEndianBytes(1f); + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2]," + + "\"data_offsets\":[0," + data.length + "]}}"; + final Path file = writeFile(dir, "model.safetensors", header, data); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("w")); + assertTrue(e.getMessage().contains("2 F32 elements"), e.getMessage()); + } + + @Test + void testTensorInfoShapeIsDefensivelyCopied() { + final int[] shape = {2, 3}; + final TensorInfo info = new TensorInfo("t", "F32", shape, 0, 24); + shape[0] = 99; + assertEquals(2, info.shape()[0], "construction must copy the caller's array"); + info.shape()[0] = 99; + assertEquals(2, info.shape()[0], "the accessor must return a copy"); + assertEquals(6, info.elementCount()); + } + + @Test + void testTensorInfoEqualsByValue() { + final TensorInfo a = new TensorInfo("t", "F32", new int[] {2, 3}, 0, 24); + final TensorInfo b = new TensorInfo("t", "F32", new int[] {2, 3}, 0, 24); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + } + + @Test + void testTensorInfoElementCountOverflowFailsLoudly() { + final TensorInfo crafted = new TensorInfo("t", "F32", + new int[] {Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE}, 0, 8); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, crafted::elementCount); + assertTrue(e.getMessage().contains("overflows"), e.getMessage()); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java index e9ebf71538..966de9e19b 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java @@ -155,4 +155,26 @@ void testRejectsMalformedHeaders(String header) { assertTrue(e.getMessage().contains("Malformed safetensors header at offset"), () -> "Message should carry the offset, got: " + e.getMessage()); } + @Test + void testSignedUnicodeEscapeFailsLoudly() { + // Integer.parseInt would accept "-0FF" and decode the wrong character; the parser must not. + final String header = "{\"__metadata__\":{\"note\":\"a\\u-0FFb\"}," + + "\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; + assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(header)); + } + + @Test + void testMalformedNumberInSkippedFieldFailsLoudly() { + // Skipped unknown fields still hold values to the JSON grammar; "1e++--..5" is not a number. + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]," + + "\"data_offsets\":[0,4],\"unknown\":1e++--..5}}"; + assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(header)); + } + + @Test + void testWellFormedNumbersInSkippedFieldsAreAccepted() { + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]," + + "\"data_offsets\":[0,4],\"a\":-1.5e+10,\"b\":0.25,\"c\":3}}"; + assertEquals(1, SafetensorsHeaderParser.parse(header).tensors().size()); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java new file mode 100644 index 0000000000..f590a4edb3 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.StringJoiner; + +/** + * Writes small well-formed safetensors fixtures for tests and benchmarks, replacing the writer + * that used to be copied into every test class. Negative tests that need deliberately malformed + * bytes still hand-roll them. + */ +final class SafetensorsTestFiles { + + private SafetensorsTestFiles() { + } + + /** One F32 tensor to write: a name, a shape, and the values in row-major order. */ + record Tensor(String name, int[] shape, float[] values) { + } + + static Tensor matrix(String name, float[][] rows) { + final int dimension = rows[0].length; + final float[] values = new float[rows.length * dimension]; + for (int r = 0; r < rows.length; r++) { + System.arraycopy(rows[r], 0, values, r * dimension, dimension); + } + return new Tensor(name, new int[] {rows.length, dimension}, values); + } + + static Tensor vector(String name, float[] values) { + return new Tensor(name, new int[] {values.length}, values); + } + + /** + * Writes a safetensors file holding the given F32 tensors, header first, data in declaration + * order. + */ + static void write(Path file, Tensor... tensors) throws IOException { + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + final StringJoiner header = new StringJoiner(",", "{", "}"); + int offset = 0; + for (final Tensor tensor : tensors) { + final ByteBuffer buffer = + ByteBuffer.allocate(tensor.values().length * Float.BYTES) + .order(ByteOrder.LITTLE_ENDIAN); + for (final float value : tensor.values()) { + buffer.putFloat(value); + } + data.writeBytes(buffer.array()); + final StringJoiner shape = new StringJoiner(",", "[", "]"); + for (final int dimension : tensor.shape()) { + shape.add(Integer.toString(dimension)); + } + final int end = offset + tensor.values().length * Float.BYTES; + header.add("\"" + tensor.name() + "\":{\"dtype\":\"F32\",\"shape\":" + shape + + ",\"data_offsets\":[" + offset + "," + end + "]}"); + offset = end; + } + final byte[] headerBytes = header.toString().getBytes(StandardCharsets.UTF_8); + final ByteBuffer out = ByteBuffer.allocate(8 + headerBytes.length + data.size()) + .order(ByteOrder.LITTLE_ENDIAN); + out.putLong(headerBytes.length); + out.put(headerBytes); + out.put(data.toByteArray()); + Files.write(file, out.array()); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java index 1ccc23abdf..840ff58e0f 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java @@ -16,11 +16,7 @@ */ package opennlp.embeddings; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -35,14 +31,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.embeddings.StaticEmbeddingModel.Casing; +import opennlp.embeddings.StaticEmbeddingModel.Normalization; + import static org.junit.jupiter.api.Assertions.assertTrue; /** * A concurrency smoke test for the {@code @ThreadSafe} claim on {@link StaticEmbeddingModel}: * one shared instance, many threads, every concurrent result compared against the * single-threaded reference computed up front. All operations are deterministic, so any - * deviation under concurrency is a thread-safety defect by definition. Mirrors the - * {@code LexiconConcurrencyTest} pattern from the opennlp-wordnet module. + * deviation under concurrency is a thread-safety defect by definition: one shared instance, + * reference results computed single-threaded first, then compared under contention. */ class StaticEmbeddingModelConcurrencyTest { @@ -57,25 +56,10 @@ private static StaticEmbeddingModel loadFixture(Path dir) throws IOException { {0f, 0f}, {0f, 0f}, {0f, 0f}, {3f, 3f}, {2f, 4f}, {2f, 1f}, {1f, 2f}, {-3f, -1f}, }; - final ByteBuffer buffer = ByteBuffer.allocate(rows.length * 2 * 4) - .order(ByteOrder.LITTLE_ENDIAN); - for (final float[] row : rows) { - for (final float value : row) { - buffer.putFloat(value); - } - } - final byte[] data = buffer.array(); - final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + rows.length - + ",2],\"data_offsets\":[0," + data.length + "]}}"; - final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) - .putLong(headerBytes.length).array()); - out.write(headerBytes); - out.write(data); final Path tensors = dir.resolve("model.safetensors"); - Files.write(tensors, out.toByteArray()); - return StaticEmbeddingModel.load(vocab, tensors, true, true); + SafetensorsTestFiles.write(tensors, SafetensorsTestFiles.matrix("embeddings", rows)); + return StaticEmbeddingModel.load(vocab, tensors, + Casing.UNCASED, Normalization.L2); } @Test diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java index b208de352c..8a553d7c08 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java @@ -28,6 +28,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.embeddings.StaticEmbeddingModel.Casing; +import opennlp.embeddings.StaticEmbeddingModel.Normalization; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -66,29 +69,14 @@ private static Path writeVocab(Path dir) throws IOException { } private static Path writeSafetensors(Path dir) throws IOException { - final ByteBuffer buffer = - ByteBuffer.allocate(ROWS.length * DIMENSION * 4).order(ByteOrder.LITTLE_ENDIAN); - for (final float[] row : ROWS) { - for (final float value : row) { - buffer.putFloat(value); - } - } - final byte[] data = buffer.array(); - final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + ROWS.length + "," - + DIMENSION + "],\"data_offsets\":[0," + data.length + "]}}"; - final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) - .putLong(headerBytes.length).array()); - out.write(headerBytes); - out.write(data); final Path file = dir.resolve("model.safetensors"); - Files.write(file, out.toByteArray()); + SafetensorsTestFiles.write(file, SafetensorsTestFiles.matrix("embeddings", ROWS)); return file; } private static StaticEmbeddingModel load(Path dir) throws IOException { - return StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir), true, false); + return StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir), + Casing.UNCASED, Normalization.NONE); } @Test @@ -237,7 +225,8 @@ void testZeroVectorRowScoresZeroNotNaN(@TempDir Path dir) throws IOException { out.write(data); final Path tensors = dir.resolve("zero-model.safetensors"); Files.write(tensors, out.toByteArray()); - final StaticEmbeddingModel model = StaticEmbeddingModel.load(vocab, tensors, true, false); + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(vocab, tensors, Casing.UNCASED, Normalization.NONE); final List result = model.mostSimilar("a", 5); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index 31c7becdfd..29b1062cc0 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -28,6 +28,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.embeddings.StaticEmbeddingModel.Casing; +import opennlp.embeddings.StaticEmbeddingModel.Normalization; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -57,51 +60,24 @@ private static Path writeVocab(Path dir) throws IOException { } private static Path writeSafetensors(Path dir, boolean withWeights) throws IOException { - final ByteArrayOutputStream data = new ByteArrayOutputStream(); - final ByteBuffer embeddingBuffer = - ByteBuffer.allocate(ROWS.length * DIMENSION * 4).order(ByteOrder.LITTLE_ENDIAN); - for (final float[] row : ROWS) { - for (final float value : row) { - embeddingBuffer.putFloat(value); - } - } - final byte[] embeddingBytes = embeddingBuffer.array(); - data.write(embeddingBytes); - - String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + ROWS.length + "," - + DIMENSION + "],\"data_offsets\":[0," + embeddingBytes.length + "]}"; + final Path file = dir.resolve("model.safetensors"); if (withWeights) { // Weight per row: [1, 1, 1, 2, 1, 1] so "hello" (row 3) counts double in the sum but not // in the pooling denominator, which is the exact behavior being pinned. - final float[] weightValues = {1f, 1f, 1f, 2f, 1f, 1f}; - final ByteBuffer weightBuffer = - ByteBuffer.allocate(weightValues.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (final float value : weightValues) { - weightBuffer.putFloat(value); - } - final byte[] weightBytes = weightBuffer.array(); - final int start = embeddingBytes.length; - data.write(weightBytes); - header += ",\"weights\":{\"dtype\":\"F32\",\"shape\":[" + weightValues.length - + "],\"data_offsets\":[" + start + "," + (start + weightBytes.length) + "]}"; + SafetensorsTestFiles.write(file, + SafetensorsTestFiles.matrix("embeddings", ROWS), + SafetensorsTestFiles.vector("weights", new float[] {1f, 1f, 1f, 2f, 1f, 1f})); + } else { + SafetensorsTestFiles.write(file, SafetensorsTestFiles.matrix("embeddings", ROWS)); } - header += "}"; - - final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) - .putLong(headerBytes.length).array()); - out.write(headerBytes); - out.write(data.toByteArray()); - final Path file = dir.resolve("model.safetensors"); - Files.write(file, out.toByteArray()); return file; } @Test void testEmbedMeanPoolsWithoutWeights(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); final float[] result = model.embed("hello world"); @@ -113,7 +89,8 @@ void testEmbedMeanPoolsWithoutWeights(@TempDir Path dir) throws IOException { void testEmbedAppliesPerTokenWeightsButDividesByTokenCount(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, true), true, false); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, true), + Casing.UNCASED, Normalization.NONE); final float[] result = model.embed("hello world"); @@ -125,7 +102,8 @@ void testEmbedAppliesPerTokenWeightsButDividesByTokenCount(@TempDir Path dir) @Test void testEmbedNormalizesToUnitLength(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, true); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.L2); final float[] result = model.embed("cat"); @@ -142,7 +120,8 @@ void testEmbedNormalizesToUnitLength(@TempDir Path dir) throws IOException { @Test void testEmbedSkipsUnknownTokens(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); // "xyzzy" cannot be represented by any vocabulary piece, so it becomes [UNK] and must be // excluded from both the sum and the pooling denominator, leaving just "cat". @@ -154,7 +133,8 @@ void testEmbedSkipsUnknownTokens(@TempDir Path dir) throws IOException { @Test void testEmbedOfTextWithNoInVocabularyTokensIsZeroVector(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed("xyzzy"), 1e-5f); } @@ -162,7 +142,8 @@ void testEmbedOfTextWithNoInVocabularyTokensIsZeroVector(@TempDir Path dir) thro @Test void testEmbedOfEmptyTextIsZeroVectorNotAnError(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, true); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.L2); assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed(""), 1e-5f); } @@ -170,7 +151,8 @@ void testEmbedOfEmptyTextIsZeroVectorNotAnError(@TempDir Path dir) throws IOExce @Test void testDimensionAndVocabularySizeAccessors(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); assertEquals(DIMENSION, model.dimension()); assertEquals(VOCAB_TOKENS.size(), model.vocabularySize()); @@ -179,7 +161,8 @@ void testDimensionAndVocabularySizeAccessors(@TempDir Path dir) throws IOExcepti @Test void testEmbedRejectsNullText(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = - StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), true, false); + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); assertThrows(IllegalArgumentException.class, () -> model.embed(null)); } @@ -190,9 +173,9 @@ void testLoadRejectsNullArguments(@TempDir Path dir) throws IOException { final Path tensors = writeSafetensors(dir, false); assertThrows(IllegalArgumentException.class, - () -> StaticEmbeddingModel.load(null, tensors, true, false)); + () -> StaticEmbeddingModel.load(null, tensors, Casing.UNCASED, Normalization.NONE)); assertThrows(IllegalArgumentException.class, - () -> StaticEmbeddingModel.load(vocab, null, true, false)); + () -> StaticEmbeddingModel.load(vocab, null, Casing.UNCASED, Normalization.NONE)); } @Test @@ -201,7 +184,8 @@ void testLoadRejectsVocabularySizeMismatch(@TempDir Path dir) throws IOException Files.write(shortVocab, List.of("[CLS]", "[SEP]", "[UNK]")); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> StaticEmbeddingModel.load(shortVocab, writeSafetensors(dir, false), true, false)); + () -> StaticEmbeddingModel.load(shortVocab, writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE)); assertTrue(e.getMessage().contains("rows")); } @@ -235,7 +219,7 @@ void testLoadRejectsWeightsSizeMismatch(@TempDir Path dir) throws IOException { Files.write(file, out.toByteArray()); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> StaticEmbeddingModel.load(writeVocab(dir), file, true, false)); + () -> StaticEmbeddingModel.load(writeVocab(dir), file, Casing.UNCASED, Normalization.NONE)); assertTrue(e.getMessage().contains("weights")); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordPieceVocabularyTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordPieceVocabularyTest.java new file mode 100644 index 0000000000..4ae76c1ba8 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordPieceVocabularyTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The vocabulary contract: line number is the id, duplicates fail loud, the id lookup uses a + * {@code -1} sentinel, and the reverse lookup enforces its bounds. + */ +class WordPieceVocabularyTest { + + @Test + void testLineNumberIsTheTokenId() { + final WordPieceVocabulary vocabulary = + WordPieceVocabulary.fromLines(List.of("[CLS]", "[SEP]", "hello", "world"), "test"); + assertEquals(4, vocabulary.size()); + assertEquals(0, vocabulary.id("[CLS]")); + assertEquals(2, vocabulary.id("hello")); + assertEquals("world", vocabulary.token(3)); + assertTrue(vocabulary.tokens().contains("hello")); + } + + @Test + void testUnknownTokenIdIsTheSentinel() { + final WordPieceVocabulary vocabulary = + WordPieceVocabulary.fromLines(List.of("hello"), "test"); + assertEquals(-1, vocabulary.id("missing")); + assertThrows(IllegalArgumentException.class, () -> vocabulary.id(null)); + } + + @Test + void testDuplicateTokenFailsLoudlyNamingBothLines() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> WordPieceVocabulary.fromLines(List.of("hello", "world", "hello"), "test")); + assertTrue(e.getMessage().contains("hello"), e.getMessage()); + assertTrue(e.getMessage().contains("0") && e.getMessage().contains("2"), e.getMessage()); + } + + @Test + void testReverseLookupEnforcesBounds() { + final WordPieceVocabulary vocabulary = + WordPieceVocabulary.fromLines(List.of("hello"), "test"); + assertEquals("hello", vocabulary.token(0)); + assertThrows(IllegalArgumentException.class, () -> vocabulary.token(-1)); + assertThrows(IllegalArgumentException.class, () -> vocabulary.token(1)); + } + + @Test + void testReadFromFileMatchesInMemoryLines(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("vocab.txt"); + Files.write(file, List.of("[CLS]", "token")); + final WordPieceVocabulary read = WordPieceVocabulary.read(file); + assertEquals(2, read.size()); + assertEquals(1, read.id("token")); + } +} From 78414d98b68667f2e479687d149cd6584eb076a1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 10 Jul 2026 16:58:02 -0400 Subject: [PATCH 34/82] OPENNLP-1877: Register opennlp-embeddings in the root dependencyManagement The distribution dependency added for the review round needs the managed version like the other extension modules; without it the distr module fails to resolve the artifact. --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index 3f77763325..9811276bd4 100644 --- a/pom.xml +++ b/pom.xml @@ -204,6 +204,12 @@ test-jar + + opennlp-embeddings + ${project.groupId} + ${project.version} + + opennlp-morfologik ${project.groupId} From 4cb03e806ba4a032af8e1d3c29470337d60e9df8 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 11 Jul 2026 10:47:34 -0400 Subject: [PATCH 35/82] OPENNLP-1877: Rename WordPieceVocabulary to WordpieceVocabulary Matches the casing of the existing public WordpieceTokenizer. --- ...ocabulary.java => WordpieceVocabulary.java} | 10 +++++----- ...yTest.java => WordpieceVocabularyTest.java} | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) rename opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/{WordPieceVocabulary.java => WordpieceVocabulary.java} (93%) rename opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/{WordPieceVocabularyTest.java => WordpieceVocabularyTest.java} (85%) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java similarity index 93% rename from opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java rename to opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java index 7cfd65d10f..b17a0c5741 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordPieceVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java @@ -37,12 +37,12 @@ *

Immutable and safe for concurrent reads after construction.

*/ @ThreadSafe -final class WordPieceVocabulary { +final class WordpieceVocabulary { private final Map idByToken; private final List tokenById; - private WordPieceVocabulary(Map idByToken, List tokenById) { + private WordpieceVocabulary(Map idByToken, List tokenById) { this.idByToken = idByToken; this.tokenById = tokenById; } @@ -56,7 +56,7 @@ private WordPieceVocabulary(Map idByToken, List tokenBy * contains a duplicate token. * @throws IOException Thrown if reading the file fails. */ - static WordPieceVocabulary read(Path file) throws IOException { + static WordpieceVocabulary read(Path file) throws IOException { if (file == null) { throw new IllegalArgumentException("File must not be null"); } @@ -67,7 +67,7 @@ static WordPieceVocabulary read(Path file) throws IOException { } // Package-private so tests can build a vocabulary from in-memory lines without a temp file. - static WordPieceVocabulary fromLines(List lines, String sourceName) { + static WordpieceVocabulary fromLines(List lines, String sourceName) { final Map idByToken = new LinkedHashMap<>(lines.size() * 2); for (int id = 0; id < lines.size(); id++) { final String token = lines.get(id); @@ -77,7 +77,7 @@ static WordPieceVocabulary fromLines(List lines, String sourceName) { + "' more than once, at lines " + idByToken.get(token) + " and " + id); } } - return new WordPieceVocabulary(Collections.unmodifiableMap(idByToken), List.copyOf(lines)); + return new WordpieceVocabulary(Collections.unmodifiableMap(idByToken), List.copyOf(lines)); } /** {@return every token in this vocabulary, suitable for a WordpieceTokenizer} */ diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordPieceVocabularyTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpieceVocabularyTest.java similarity index 85% rename from opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordPieceVocabularyTest.java rename to opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpieceVocabularyTest.java index 4ae76c1ba8..0609efa834 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordPieceVocabularyTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpieceVocabularyTest.java @@ -32,12 +32,12 @@ * The vocabulary contract: line number is the id, duplicates fail loud, the id lookup uses a * {@code -1} sentinel, and the reverse lookup enforces its bounds. */ -class WordPieceVocabularyTest { +class WordpieceVocabularyTest { @Test void testLineNumberIsTheTokenId() { - final WordPieceVocabulary vocabulary = - WordPieceVocabulary.fromLines(List.of("[CLS]", "[SEP]", "hello", "world"), "test"); + final WordpieceVocabulary vocabulary = + WordpieceVocabulary.fromLines(List.of("[CLS]", "[SEP]", "hello", "world"), "test"); assertEquals(4, vocabulary.size()); assertEquals(0, vocabulary.id("[CLS]")); assertEquals(2, vocabulary.id("hello")); @@ -47,8 +47,8 @@ void testLineNumberIsTheTokenId() { @Test void testUnknownTokenIdIsTheSentinel() { - final WordPieceVocabulary vocabulary = - WordPieceVocabulary.fromLines(List.of("hello"), "test"); + final WordpieceVocabulary vocabulary = + WordpieceVocabulary.fromLines(List.of("hello"), "test"); assertEquals(-1, vocabulary.id("missing")); assertThrows(IllegalArgumentException.class, () -> vocabulary.id(null)); } @@ -56,15 +56,15 @@ void testUnknownTokenIdIsTheSentinel() { @Test void testDuplicateTokenFailsLoudlyNamingBothLines() { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> WordPieceVocabulary.fromLines(List.of("hello", "world", "hello"), "test")); + () -> WordpieceVocabulary.fromLines(List.of("hello", "world", "hello"), "test")); assertTrue(e.getMessage().contains("hello"), e.getMessage()); assertTrue(e.getMessage().contains("0") && e.getMessage().contains("2"), e.getMessage()); } @Test void testReverseLookupEnforcesBounds() { - final WordPieceVocabulary vocabulary = - WordPieceVocabulary.fromLines(List.of("hello"), "test"); + final WordpieceVocabulary vocabulary = + WordpieceVocabulary.fromLines(List.of("hello"), "test"); assertEquals("hello", vocabulary.token(0)); assertThrows(IllegalArgumentException.class, () -> vocabulary.token(-1)); assertThrows(IllegalArgumentException.class, () -> vocabulary.token(1)); @@ -74,7 +74,7 @@ void testReverseLookupEnforcesBounds() { void testReadFromFileMatchesInMemoryLines(@TempDir Path dir) throws IOException { final Path file = dir.resolve("vocab.txt"); Files.write(file, List.of("[CLS]", "token")); - final WordPieceVocabulary read = WordPieceVocabulary.read(file); + final WordpieceVocabulary read = WordpieceVocabulary.read(file); assertEquals(2, read.size()); assertEquals(1, read.id("token")); } From e76555e9dbe1e62ef57cb63772327b985b75a07c Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 11 Jul 2026 10:48:04 -0400 Subject: [PATCH 36/82] OPENNLP-1877: Add the TextEmbedder seam; both engines implement it TextEmbedder in opennlp-api is the text-level embedding contract: embed(CharSequence), embedAll (default loops one at a time; runtimes that batch efficiently should override), and dimension(). It is the text-level counterpart of the word-level WordVectorTable, and the javadoc states the layer difference. StaticEmbeddingModel implements it with a thin CharSequence overload; the existing embed(String) hot path is untouched. SentenceVectorsDL implements it as the contextual tier: getVectors stays the primary entry point unchanged, embed adapts it (OrtException wrapped unchecked, the seam is runtime-neutral), and dimension() reads the model's declared output metadata with a one-time probe fallback for dynamic shapes. A batched embedAll override remains open as a follow-up. The adapter test drives a real ONNX session: a committed 373-byte model (generation script alongside) computes token_id times a fixed weight row, so every expected vector is hand-computable. The gitignored *.onnx pattern is force-added for this one bundled fixture; rat-excludes covers the binary. --- .../tools/embeddings/TextEmbedder.java | 70 ++++++++++++++ .../opennlp/dl/vectors/SentenceVectorsDL.java | 74 ++++++++++++++- .../SentenceVectorsDLEmbedderTest.java | 89 ++++++++++++++++++ .../dl/vectors/gen_tiny_vectors_model.py | 57 +++++++++++ .../opennlp/dl/vectors/tiny-vectors.onnx | Bin 0 -> 373 bytes .../embeddings/StaticEmbeddingModel.java | 26 ++++- .../embeddings/StaticEmbeddingModelTest.java | 23 +++++ rat-excludes | 2 + 8 files changed, 336 insertions(+), 5 deletions(-) create mode 100644 opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/gen_tiny_vectors_model.py create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/tiny-vectors.onnx diff --git a/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java new file mode 100644 index 0000000000..e5f5781f85 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.embeddings; + +import java.util.List; + +/** + * Encodes a piece of text into a single fixed-length vector. + * + *

A text embedder maps whole texts (a sentence, a paragraph, a document) into one dense + * vector whose geometry carries meaning: texts about the same thing land near each other. This + * is the text-level counterpart of {@link opennlp.tools.util.wordvector.WordVectorTable}, which + * looks up a stored vector for a single word; an embedder composes a vector for text it has + * never seen, handling tokenization and pooling internally.

+ * + *

Implementations are expected to be safe for concurrent use by multiple threads; any + * implementation that is not must document it. Implementation failures during encoding (a + * backing runtime error, a corrupted model) surface as unchecked exceptions carrying the + * underlying cause.

+ */ +public interface TextEmbedder { + + /** + * Embeds a piece of text. + * + * @param text The text to embed; must not be null. + * @return The embedding vector, of length {@link #dimension()}. + * @throws IllegalArgumentException Thrown if {@code text} is null. + */ + float[] embed(CharSequence text); + + /** + * Embeds several texts. + * + *

The default implementation embeds one text at a time. Implementations backed by a + * runtime that executes batches more efficiently than single inputs should override this + * method.

+ * + * @param texts The texts to embed; must not be null and must not contain null. + * @return One embedding vector per input, in input order. + * @throws IllegalArgumentException Thrown if {@code texts} is null or contains null. + */ + default float[][] embedAll(List texts) { + if (texts == null) { + throw new IllegalArgumentException("Texts must not be null"); + } + final float[][] vectors = new float[texts.size()][]; + for (int i = 0; i < vectors.length; i++) { + vectors[i] = embed(texts.get(i)); + } + return vectors; + } + + /** {@return the dimension of every vector this embedder produces} */ + int dimension(); +} diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java index f1250ea601..7aa6613bcf 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java @@ -24,13 +24,16 @@ import java.util.HashMap; import java.util.Map; +import ai.onnxruntime.NodeInfo; import ai.onnxruntime.OnnxTensor; import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import ai.onnxruntime.TensorInfo; import opennlp.dl.AbstractDL; import opennlp.dl.Tokens; import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.embeddings.TextEmbedder; import opennlp.tools.tokenize.Tokenizer; @@ -56,9 +59,19 @@ * holds no per-call instance state and the underlying {@link OrtSession} supports * concurrent execution. This thread-safety guarantee applies until {@link #close()} * is called; callers must not race {@code close()} with inference methods.

+ * + *

As a {@link TextEmbedder} this class is the contextual tier: every vector comes from a + * full transformer forward pass. {@link #getVectors(String)} remains the primary entry point + * and is unchanged; {@link #embed(CharSequence)} is an adapter over it for callers coding + * against the seam. Batched inference ({@code embedAll} executing one padded model run) is a + * possible future override; the inherited default embeds one text at a time.

*/ @ThreadSafe -public class SentenceVectorsDL extends AbstractDL { +public class SentenceVectorsDL extends AbstractDL implements TextEmbedder { + + // The hidden dimension declared by the model's output metadata, or a value <= 0 when the + // model declares it dynamically; dimension() then probes once and caches here. + private volatile int dimension; /** * Instantiates a {@link SentenceVectorsDL sentence vector generator} for an @@ -94,6 +107,7 @@ public SentenceVectorsDL(final File model, final File vocabulary, final boolean throws OrtException, IOException { super(model, vocabulary, new OrtSession.SessionOptions(), lowerCase); + this.dimension = declaredOutputDimension(session); } @@ -132,6 +146,64 @@ public float[] getVectors(final String sentence) throws OrtException { } + /** + * Embeds a piece of text. This is {@link #getVectors(String)} behind the + * {@link TextEmbedder} contract: inference failures surface as an unchecked exception + * because the seam is runtime-neutral. + * + * @param text The text to embed; must not be {@code null}. + * @return The sentence vector, of length {@link #dimension()}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + * @throws IllegalStateException Thrown if inference fails; the cause carries the + * underlying {@link OrtException}. + */ + @Override + public float[] embed(final CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("Text must not be null"); + } + try { + return getVectors(text instanceof String s ? s : text.toString()); + } catch (OrtException e) { + throw new IllegalStateException("Sentence vector inference failed.", e); + } + } + + /** + * {@return the dimension of every vector this model produces} Read from the model's + * declared output metadata when it is static there; a model that declares the hidden + * dimension dynamically is probed with one inference on first call and the result cached. + */ + @Override + public int dimension() { + final int declared = dimension; + if (declared > 0) { + return declared; + } + synchronized (this) { + if (dimension <= 0) { + dimension = embed("a").length; + } + return dimension; + } + } + + // The last dimension of the first output's declared shape; getVectors reads the first + // output, so only its shape matters. Returns -1 when the model declares it dynamically. + private static int declaredOutputDimension(final OrtSession session) throws OrtException { + for (final NodeInfo output : session.getOutputInfo().values()) { + if (output.getInfo() instanceof TensorInfo tensorInfo) { + final long[] shape = tensorInfo.getShape(); + final long last = shape.length > 0 ? shape[shape.length - 1] : -1; + if (last > 0 && last <= Integer.MAX_VALUE) { + return (int) last; + } + } + return -1; + } + return -1; + } + /** * Encodes text as model inputs: wordpiece token ids, an attention mask of ones, * and single-segment (all zero) token type ids. diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java new file mode 100644 index 0000000000..ee2920ec79 --- /dev/null +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.dl.vectors; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.embeddings.TextEmbedder; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * The {@link TextEmbedder} adapter driven through a real ONNX session. The bundled + * {@code tiny-vectors.onnx} (see {@code gen_tiny_vectors_model.py} next to it) computes + * {@code output[b][t] = float(input_ids[b][t]) * [0.5, -1, 2]}, so every expected vector is + * hand-computable from the vocabulary ids: {@code getVectors} returns the vector at the + * {@code [CLS]} position, and {@code [CLS]} sits at line 7 of the test vocabulary. + */ +class SentenceVectorsDLEmbedderTest { + + // 7 * [0.5, -1, 2] + private static final float[] CLS_VECTOR = {3.5f, -7f, 14f}; + + private static File model() throws URISyntaxException { + return new File(SentenceVectorsDLEmbedderTest.class + .getResource("/opennlp/dl/vectors/tiny-vectors.onnx").toURI()); + } + + private static File vocab(Path dir) throws IOException { + final Path file = dir.resolve("vocab.txt"); + // Line number = id: [UNK]=2, [SEP]=3, hello=4, world=5, [CLS]=7. + Files.write(file, List.of("[PAD]", "unused1", "[UNK]", "[SEP]", "hello", "world", + "unused2", "[CLS]")); + return file.toFile(); + } + + @Test + void testEmbedderContractOverARealSession(@TempDir Path dir) throws Exception { + try (SentenceVectorsDL vectors = new SentenceVectorsDL(model(), vocab(dir))) { + + // The original entry point is untouched by the interface adoption. + assertArrayEquals(CLS_VECTOR, vectors.getVectors("hello world"), 1e-5f); + + final TextEmbedder embedder = vectors; + + // The dimension comes from the model's declared output metadata, no inference needed. + assertEquals(3, embedder.dimension()); + + // The seam produces the same vector as the original entry point, for String and + // non-String inputs alike. + assertArrayEquals(CLS_VECTOR, embedder.embed("hello world"), 1e-5f); + assertArrayEquals(CLS_VECTOR, embedder.embed(new StringBuilder("hello world")), 1e-5f); + + // The inherited default batch method returns one vector per input, in input order; + // this model's [CLS]-position output is input-independent by construction. + final float[][] batch = embedder.embedAll(List.of("hello world", "hello")); + assertEquals(2, batch.length); + assertArrayEquals(CLS_VECTOR, batch[0], 1e-5f); + assertArrayEquals(CLS_VECTOR, batch[1], 1e-5f); + + assertThrows(IllegalArgumentException.class, () -> embedder.embed(null)); + assertThrows(IllegalArgumentException.class, () -> embedder.embedAll(null)); + } + } +} diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/gen_tiny_vectors_model.py b/opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/gen_tiny_vectors_model.py new file mode 100644 index 0000000000..05a67ccd5f --- /dev/null +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/gen_tiny_vectors_model.py @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates tiny-vectors.onnx, the deterministic model behind SentenceVectorsDLEmbedderTest. +# +# The graph computes output[b][t][d] = float(input_ids[b][t]) * W[0][d] with +# W = [[0.5, -1.0, 2.0]], so the vector at any token position is that token's vocabulary id +# times W, hand-computable in the test. It declares the same three inputs a BERT-style +# encoder declares (input_ids, attention_mask, token_type_ids; the latter two are accepted +# and ignored) and one output of shape [batch, tokens, 3] so the hidden dimension is static +# in the model metadata. +# +# Regenerate with: python3 gen_tiny_vectors_model.py (requires the onnx package) + +import numpy as np +import onnx +from onnx import TensorProto, helper, numpy_helper + +W = np.array([[0.5, -1.0, 2.0]], dtype=np.float32) + +cast = helper.make_node("Cast", ["input_ids"], ["ids_float"], to=TensorProto.FLOAT) +unsqueeze = helper.make_node("Unsqueeze", ["ids_float", "axes"], ["ids_3d"]) +matmul = helper.make_node("MatMul", ["ids_3d", "w"], ["last_hidden_state"]) + + +def encoder_input(name): + return helper.make_tensor_value_info(name, TensorProto.INT64, ["batch", "tokens"]) + + +graph = helper.make_graph( + [cast, unsqueeze, matmul], + "tiny-vectors", + [encoder_input("input_ids"), encoder_input("attention_mask"), + encoder_input("token_type_ids")], + [helper.make_tensor_value_info( + "last_hidden_state", TensorProto.FLOAT, ["batch", "tokens", 3])], + [numpy_helper.from_array(np.array([2], dtype=np.int64), name="axes"), + numpy_helper.from_array(W, name="w")], +) + +model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) +model.ir_version = 8 +onnx.checker.check_model(model) +onnx.save(model, "tiny-vectors.onnx") +print("wrote tiny-vectors.onnx,", len(model.SerializeToString()), "bytes") diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/tiny-vectors.onnx b/opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/tiny-vectors.onnx new file mode 100644 index 0000000000000000000000000000000000000000..7d63c91322d8152357f9dd203f1fc88a96a9df4b GIT binary patch literal 373 zcmaJ-O-sW-5Y295vKgwxRfr;9f>elj(VG_~-U>Z>>1EkABV8JIt2?pQlRw3O;{Wki z7^8yV#~fzhy*H0HBgEM&heJq=T{NUj<~bzM76d3dQk6$x;1JM5WpIB11mEQI<2 zy^P+0(<)D{tiNp#9}x!d2?ATn&ARnaGgDP)l_yP@M */ @ThreadSafe -public final class StaticEmbeddingModel { +public final class StaticEmbeddingModel implements TextEmbedder { /** How the tokenizer treats letter case, matching the base model's tokenizer configuration. */ public enum Casing { @@ -90,7 +91,7 @@ public enum Normalization { private final float[] embeddings; private final float[] weights; private final int dimension; - private final WordPieceVocabulary vocabulary; + private final WordpieceVocabulary vocabulary; private final BertTokenizer tokenizer; private final boolean normalize; private final String unknownToken; @@ -100,7 +101,7 @@ public enum Normalization { private final boolean[] specialRows; private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, - WordPieceVocabulary vocabulary, BertTokenizer tokenizer, + WordpieceVocabulary vocabulary, BertTokenizer tokenizer, boolean normalize, String unknownToken, double[] rowNorms, boolean[] specialRows) { this.embeddings = embeddings; @@ -231,7 +232,7 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } final boolean lowerCase = casing == Casing.UNCASED; final boolean normalize = normalization == Normalization.L2; - final WordPieceVocabulary vocabulary = WordPieceVocabulary.read(vocabularyFile); + final WordpieceVocabulary vocabulary = WordpieceVocabulary.read(vocabularyFile); final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); final String matrixName = tensors.singleMatrixTensorName(); @@ -278,6 +279,22 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil normalize, WordpieceTokenizer.BERT_UNK_TOKEN, rowNorms, specialRows); } + /** + * Embeds a piece of text. + * + * @param text The text to embed. Must not be {@code null}. + * @return The pooled embedding vector, of length {@link #dimension()}. A text with no + * in-vocabulary tokens yields a zero vector. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + @Override + public float[] embed(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("Text must not be null"); + } + return embed(text instanceof String s ? s : text.toString()); + } + /** * Embeds a piece of text. * @@ -337,6 +354,7 @@ public float[] embed(String text) { } /** {@return the dimension of every vector this model produces} */ + @Override public int dimension() { return dimension; } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index 29b1062cc0..6ebc06a049 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -30,6 +30,7 @@ import opennlp.embeddings.StaticEmbeddingModel.Casing; import opennlp.embeddings.StaticEmbeddingModel.Normalization; +import opennlp.tools.embeddings.TextEmbedder; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -308,4 +309,26 @@ void testDirectoryLoadRejectsContradictoryStripAccents(@TempDir Path dir) throws assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); assertTrue(e.getMessage().contains("strip_accents")); } + + @Test + void testTextEmbedderSeamMatchesDirectUseAndBatches(@TempDir Path dir) throws IOException { + final TextEmbedder embedder = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); + + // The CharSequence entry point produces the same vector as the String one, including for a + // CharSequence that is not a String. + assertArrayEquals(new float[] {3.5f, 35f, 350f}, + embedder.embed(new StringBuilder("hello world")), 1e-5f); + assertEquals(DIMENSION, embedder.dimension()); + + // The interface's default batch method returns one vector per input, in input order. + final float[][] vectors = embedder.embedAll(List.of("hello world", "cat")); + assertEquals(2, vectors.length); + assertArrayEquals(new float[] {3.5f, 35f, 350f}, vectors[0], 1e-5f); + assertArrayEquals(new float[] {5f, 50f, 500f}, vectors[1], 1e-5f); + + assertThrows(IllegalArgumentException.class, () -> embedder.embed(null)); + assertThrows(IllegalArgumentException.class, () -> embedder.embedAll(null)); + } } diff --git a/rat-excludes b/rat-excludes index a505ab2d60..4bc20b38ad 100644 --- a/rat-excludes +++ b/rat-excludes @@ -71,6 +71,8 @@ src/main/resources/opennlp/tools/tokenize/uax29/ExtendedPictographic.txt src/main/resources/opennlp/tools/util/normalizer/confusables.txt src/test/resources/opennlp/tools/tokenize/uax29/WordBreakTest.txt + +src/test/resources/opennlp/dl/vectors/tiny-vectors.onnx src/test/resources/opennlp/subword/sentencepiece/*.model src/test/resources/opennlp/subword/sentencepiece/*.fixtures.tsv From d43f6e14d260d136d4e9c9501d09edc792467421 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 09:21:10 -0400 Subject: [PATCH 37/82] OPENNLP-1877: Give the embeddings module its own tokenization pipeline WordpiecePipeline is a module-internal copy of the five-stage BERT pipeline over WordpieceTokenizer, pinned by the reference token sequences. StaticEmbeddingModel uses it instead of a shared class, so the hot path is unchanged operation for operation and the module no longer depends on any tokenizer being reworked elsewhere. The SentenceVectorsDL class javadoc is reduced to the factual contract. --- .../opennlp/dl/vectors/SentenceVectorsDL.java | 8 +- .../embeddings/StaticEmbeddingModel.java | 7 +- .../opennlp/embeddings/WordpiecePipeline.java | 156 ++++++++++++++++++ .../embeddings/WordpiecePipelineTest.java | 138 ++++++++++++++++ 4 files changed, 300 insertions(+), 9 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java index 7aa6613bcf..68c55672c5 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java @@ -60,11 +60,9 @@ * concurrent execution. This thread-safety guarantee applies until {@link #close()} * is called; callers must not race {@code close()} with inference methods.

* - *

As a {@link TextEmbedder} this class is the contextual tier: every vector comes from a - * full transformer forward pass. {@link #getVectors(String)} remains the primary entry point - * and is unchanged; {@link #embed(CharSequence)} is an adapter over it for callers coding - * against the seam. Batched inference ({@code embedAll} executing one padded model run) is a - * possible future override; the inherited default embeds one text at a time.

+ *

{@link #getVectors(String)} is the primary entry point; {@link #embed(CharSequence)} + * adapts it to the {@link TextEmbedder} contract. The inherited {@code embedAll} embeds one + * text at a time.

*/ @ThreadSafe public class SentenceVectorsDL extends AbstractDL implements TextEmbedder { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 35ea6783fc..1446334947 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -26,7 +26,6 @@ import opennlp.tools.commons.ThreadSafe; import opennlp.tools.embeddings.TextEmbedder; -import opennlp.tools.tokenize.BertTokenizer; import opennlp.tools.tokenize.WordpieceTokenizer; /** @@ -92,7 +91,7 @@ public enum Normalization { private final float[] weights; private final int dimension; private final WordpieceVocabulary vocabulary; - private final BertTokenizer tokenizer; + private final WordpiecePipeline tokenizer; private final boolean normalize; private final String unknownToken; // Per-row L2 norms and the special-token mask are constants of the model, precomputed at @@ -101,7 +100,7 @@ public enum Normalization { private final boolean[] specialRows; private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, - WordpieceVocabulary vocabulary, BertTokenizer tokenizer, + WordpieceVocabulary vocabulary, WordpiecePipeline tokenizer, boolean normalize, String unknownToken, double[] rowNorms, boolean[] specialRows) { this.embeddings = embeddings; @@ -274,7 +273,7 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } } - final BertTokenizer tokenizer = new BertTokenizer(vocabulary.tokens(), lowerCase); + final WordpiecePipeline tokenizer = new WordpiecePipeline(vocabulary.tokens(), lowerCase); return new StaticEmbeddingModel(embeddings, weights, dimension, vocabulary, tokenizer, normalize, WordpieceTokenizer.BERT_UNK_TOKEN, rowNorms, specialRows); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java new file mode 100644 index 0000000000..ac96f205c5 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.text.Normalizer; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +import opennlp.tools.tokenize.WordpieceTokenizer; + +/** + * The full BERT tokenization pipeline used for embedding-table lookup: basic tokenization + * (control removal, whitespace normalization, CJK isolation, optional lower casing with + * accent stripping, punctuation isolation) followed by {@link WordpieceTokenizer} with the + * BERT special tokens and the reference 100-character word limit. + * + *

This module-internal pipeline exists so the embedding hot path depends only on the + * stable wordpiece stage; it produces pieces without offset bookkeeping, which the lookup + * path does not need.

+ */ +final class WordpiecePipeline { + + private static final int MAX_WORD_CHARACTERS = 100; + + private final WordpieceTokenizer wordpieceTokenizer; + private final boolean lowerCase; + + WordpiecePipeline(Set vocabulary, boolean lowerCase) { + Objects.requireNonNull(vocabulary, "vocabulary must not be null"); + this.wordpieceTokenizer = new WordpieceTokenizer(vocabulary, + WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, + WordpieceTokenizer.BERT_UNK_TOKEN, MAX_WORD_CHARACTERS); + this.lowerCase = lowerCase; + } + + String[] tokenize(String text) { + return wordpieceTokenizer.tokenize(normalize(text)); + } + + private String normalize(String text) { + String normalized = cleanText(text); + normalized = isolateCjkCharacters(normalized); + if (lowerCase) { + normalized = stripAccents(normalized.toLowerCase(Locale.ROOT)); + } + return isolatePunctuation(normalized); + } + + private static String cleanText(String text) { + final StringBuilder cleaned = new StringBuilder(text.length()); + text.codePoints().forEach(codePoint -> { + if (codePoint == 0 || codePoint == 0xFFFD || isControl(codePoint)) { + return; + } + if (isWhitespace(codePoint)) { + cleaned.append(' '); + } else { + cleaned.appendCodePoint(codePoint); + } + }); + return cleaned.toString(); + } + + private static String isolateCjkCharacters(String text) { + final StringBuilder spaced = new StringBuilder(text.length()); + text.codePoints().forEach(codePoint -> { + if (isCjk(codePoint)) { + spaced.append(' ').appendCodePoint(codePoint).append(' '); + } else { + spaced.appendCodePoint(codePoint); + } + }); + return spaced.toString(); + } + + private static String stripAccents(String text) { + final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); + final StringBuilder stripped = new StringBuilder(decomposed.length()); + decomposed.codePoints().forEach(codePoint -> { + if (Character.getType(codePoint) != Character.NON_SPACING_MARK) { + stripped.appendCodePoint(codePoint); + } + }); + return stripped.toString(); + } + + private static String isolatePunctuation(String text) { + final StringBuilder spaced = new StringBuilder(text.length()); + text.codePoints().forEach(codePoint -> { + if (isPunctuation(codePoint)) { + spaced.append(' ').appendCodePoint(codePoint).append(' '); + } else { + spaced.appendCodePoint(codePoint); + } + }); + return spaced.toString(); + } + + private static boolean isControl(int codePoint) { + if (codePoint == '\t' || codePoint == '\n' || codePoint == '\r') { + return false; + } + return switch (Character.getType(codePoint)) { + case Character.CONTROL, Character.FORMAT, Character.SURROGATE, + Character.PRIVATE_USE, Character.UNASSIGNED -> true; + default -> false; + }; + } + + private static boolean isWhitespace(int codePoint) { + if (codePoint == ' ' || codePoint == '\t' || codePoint == '\n' || codePoint == '\r') { + return true; + } + return Character.getType(codePoint) == Character.SPACE_SEPARATOR; + } + + private static boolean isPunctuation(int codePoint) { + if ((codePoint >= 33 && codePoint <= 47) || (codePoint >= 58 && codePoint <= 64) + || (codePoint >= 91 && codePoint <= 96) || (codePoint >= 123 && codePoint <= 126)) { + return true; + } + return switch (Character.getType(codePoint)) { + case Character.CONNECTOR_PUNCTUATION, Character.DASH_PUNCTUATION, + Character.START_PUNCTUATION, Character.END_PUNCTUATION, + Character.INITIAL_QUOTE_PUNCTUATION, Character.FINAL_QUOTE_PUNCTUATION, + Character.OTHER_PUNCTUATION -> true; + default -> false; + }; + } + + private static boolean isCjk(int codePoint) { + return (codePoint >= 0x4E00 && codePoint <= 0x9FFF) + || (codePoint >= 0x3400 && codePoint <= 0x4DBF) + || (codePoint >= 0x20000 && codePoint <= 0x2A6DF) + || (codePoint >= 0x2A700 && codePoint <= 0x2B73F) + || (codePoint >= 0x2B740 && codePoint <= 0x2B81F) + || (codePoint >= 0x2B820 && codePoint <= 0x2CEAF) + || (codePoint >= 0xF900 && codePoint <= 0xFAFF) + || (codePoint >= 0x2F800 && codePoint <= 0x2FA1F); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java new file mode 100644 index 0000000000..2569d5f0a1 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.Set; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Pins the module-internal tokenization pipeline against reference token sequences. + *

+ * All expected sequences were generated with the HuggingFace {@code tokenizers} reference + * implementation ({@code BertWordPieceTokenizer}) using the same vocabulary, so the lookup + * path is verified to be identical to the reference BERT tokenization. + */ +class WordpiecePipelineTest { + + private static final Set VOCABULARY = Set.of( + "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog", + "em", "##bed", "##ding", "##s", + "wurttemberg", "strasse", "grosse", + "don", "t", "wait", "what", ".", ",", "?", "!", "'", + "\u6211", "\u7231", // CJK + "natural", "language", "processing"); + + @Test + void testLowerCasesCapitalizedWords() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + final String[] tokens = + pipeline.tokenize("The quick brown fox jumps over the lazy dog."); + + final String[] expected = {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over", + "the", "lazy", "dog", ".", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testLowerCasesBeforeWordpieceSplitting() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + final String[] tokens = pipeline.tokenize("Embeddings"); + + final String[] expected = {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testStripsAccentsButKeepsNonCombiningCharacters() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped; + // the sharp s is not a combining mark and must survive, leaving an OOV token. + final String[] tokens = pipeline.tokenize("W\u00fcrttemberg Stra\u00dfe"); + + final String[] expected = {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testSplitsPunctuationRunsIntoSingleCharacters() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + final String[] tokens = pipeline.tokenize("Wait... what?!"); + + final String[] expected = {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testSplitsApostrophesAsPunctuation() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + final String[] tokens = pipeline.tokenize("don't"); + + final String[] expected = {"[CLS]", "don", "'", "t", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testIsolatesCjkIdeographs() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + final String[] tokens = pipeline.tokenize("\u6211\u7231natural language processing"); + + final String[] expected = {"[CLS]", "\u6211", "\u7231", "natural", "language", + "processing", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testCleansControlCharactersAndNormalizesWhitespace() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + // Tab and no-break space are whitespace; the NUL character is removed, + // joining "brown" and "fox" into one out-of-vocabulary token. + final String[] tokens = pipeline.tokenize("the\tquick\u00a0brown\u0000fox"); + + final String[] expected = {"[CLS]", "the", "quick", "[UNK]", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testRemovesPrivateUseAndUnassignedCharacters() { + final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); + // The reference implementation treats all C* categories as control + // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn) + // are removed, joining the surrounding text into one OOV token. + final String[] tokens = pipeline.tokenize("fox\ue000jumps and fox\ufdd0jumps"); + + final String[] expected = {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testCasedModeKeepsCaseAndAccents() { + final WordpiecePipeline pipeline = new WordpiecePipeline( + Set.of("The", "W\u00fcrttemberg", "fox"), false); + final String[] tokens = pipeline.tokenize("The W\u00fcrttemberg fox"); + + final String[] expected = + {"[CLS]", "The", "W\u00fcrttemberg", "fox", "[SEP]"}; + Assertions.assertArrayEquals(expected, tokens); + } + + @Test + void testRejectsNullVocabulary() { + Assertions.assertThrows(NullPointerException.class, () -> new WordpiecePipeline(null, true)); + } +} From 4aea42b17a8568a208766b43f4833a71c86b39c3 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 10:01:43 -0400 Subject: [PATCH 38/82] OPENNLP-1877: Load the tiny-vectors test model from the classpath stream SentenceVectorsDLEmbedderTest resolved the ONNX model with new File(url.toURI()), which only works when the resource sits on the filesystem. When the test runs from the opennlp-dl test-jar (as it does in opennlp-dl-gpu) the resource URI points inside a jar and is not hierarchical, so new File(uri) threw IllegalArgumentException and failed the opennlp-dl-gpu build. Copy the model out of the classpath into the JUnit temp dir, matching the existing pattern in LoadVocabTest. --- .../SentenceVectorsDLEmbedderTest.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java index ee2920ec79..98093d5de8 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java @@ -19,10 +19,12 @@ import java.io.File; import java.io.IOException; -import java.net.URISyntaxException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.List; +import java.util.Objects; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -45,9 +47,16 @@ class SentenceVectorsDLEmbedderTest { // 7 * [0.5, -1, 2] private static final float[] CLS_VECTOR = {3.5f, -7f, 14f}; - private static File model() throws URISyntaxException { - return new File(SentenceVectorsDLEmbedderTest.class - .getResource("/opennlp/dl/vectors/tiny-vectors.onnx").toURI()); + // Copy the model out of the classpath rather than resolving it in place: when this test runs + // from the opennlp-dl test-jar (as it does in opennlp-dl-gpu) the resource URI is inside a jar + // and is not hierarchical, so new File(uri) would fail. + private static File model(Path dir) throws IOException { + final Path file = dir.resolve("tiny-vectors.onnx"); + try (InputStream is = Objects.requireNonNull(SentenceVectorsDLEmbedderTest.class + .getResourceAsStream("/opennlp/dl/vectors/tiny-vectors.onnx"))) { + Files.copy(is, file, StandardCopyOption.REPLACE_EXISTING); + } + return file.toFile(); } private static File vocab(Path dir) throws IOException { @@ -60,7 +69,7 @@ private static File vocab(Path dir) throws IOException { @Test void testEmbedderContractOverARealSession(@TempDir Path dir) throws Exception { - try (SentenceVectorsDL vectors = new SentenceVectorsDL(model(), vocab(dir))) { + try (SentenceVectorsDL vectors = new SentenceVectorsDL(model(dir), vocab(dir))) { // The original entry point is untouched by the interface adoption. assertArrayEquals(CLS_VECTOR, vectors.getVectors("hello world"), 1e-5f); From b7ca1b671ec1085b967461bbf7be5a60512eda05 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 15:32:35 -0400 Subject: [PATCH 39/82] OPENNLP-1877: Trim commentary and tighten javadoc per review conventions --- .../opennlp/dl/vectors/SentenceVectorsDL.java | 25 +-- .../opennlp/embeddings/FlatJsonFields.java | 8 +- .../java/opennlp/embeddings/JsonCursor.java | 46 +++++- .../opennlp/embeddings/SafetensorsFile.java | 47 +++--- .../embeddings/SafetensorsHeaderParser.java | 29 ++-- .../embeddings/StaticEmbeddingModel.java | 152 ++++++++++-------- .../java/opennlp/embeddings/TensorInfo.java | 3 + .../opennlp/embeddings/WordpiecePipeline.java | 28 +++- .../embeddings/WordpieceVocabulary.java | 17 +- 9 files changed, 222 insertions(+), 133 deletions(-) diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java index 68c55672c5..a935a798c7 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java @@ -145,12 +145,10 @@ public float[] getVectors(final String sentence) throws OrtException { } /** - * Embeds a piece of text. This is {@link #getVectors(String)} behind the - * {@link TextEmbedder} contract: inference failures surface as an unchecked exception - * because the seam is runtime-neutral. + * {@inheritDoc} + * + *

Adapts {@link #getVectors(String)} to the {@link TextEmbedder} contract.

* - * @param text The text to embed; must not be {@code null}. - * @return The sentence vector, of length {@link #dimension()}. * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. * @throws IllegalStateException Thrown if inference fails; the cause carries the * underlying {@link OrtException}. @@ -168,9 +166,11 @@ public float[] embed(final CharSequence text) { } /** - * {@return the dimension of every vector this model produces} Read from the model's - * declared output metadata when it is static there; a model that declares the hidden - * dimension dynamically is probed with one inference on first call and the result cached. + * {@inheritDoc} + * + *

Read from the model's declared output metadata when it is static; a model that declares + * the hidden dimension dynamically is probed with one inference on the first call and the + * result cached.

*/ @Override public int dimension() { @@ -186,8 +186,13 @@ public int dimension() { } } - // The last dimension of the first output's declared shape; getVectors reads the first - // output, so only its shape matters. Returns -1 when the model declares it dynamically. + /** + * {@return the last dimension of the first output's declared shape, or {@code -1} when the + * model declares it dynamically} + * + * @param session The model's ONNX session. + * @throws OrtException Thrown if reading the output metadata fails. + */ private static int declaredOutputDimension(final OrtSession session) throws OrtException { for (final NodeInfo output : session.getOutputInfo().values()) { if (output.getInfo() instanceof TensorInfo tensorInfo) { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java index 81fb969676..489ebe6afa 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java @@ -22,11 +22,9 @@ /** * Reads single top-level fields out of a small flat JSON configuration file (a model's - * {@code config.json} or {@code tokenizer_config.json}) without a JSON library dependency, - * sharing {@link JsonCursor}'s scanning primitives with the safetensors header parser. Only - * what the model-directory loader needs is implemented: top-level boolean look-ups. Every - * other field, of any type and nesting, is skipped structurally, and nested occurrences of the - * looked-up name never match (a top-level field is what the configuration formats define). + * {@code config.json} or {@code tokenizer_config.json}) without a JSON library dependency. Only + * top-level boolean look-ups are implemented; every other field is skipped structurally, and a + * nested occurrence of the looked-up name never matches. */ final class FlatJsonFields { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java index 237b6e411b..9e76ee23dd 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java @@ -40,12 +40,18 @@ final class JsonCursor { this.inputName = inputName; } + /** Advances the cursor past any run of whitespace. */ void skipWhitespace() { while (position < text.length() && Character.isWhitespace(text.charAt(position))) { position++; } } + /** + * {@return the character at the cursor without advancing} + * + * @throws IllegalArgumentException Thrown if the cursor is at the end of the input. + */ char peek() { if (position >= text.length()) { throw malformed("Unexpected end of input"); @@ -53,12 +59,23 @@ char peek() { return text.charAt(position); } + /** + * {@return the character at the cursor, advancing past it} + * + * @throws IllegalArgumentException Thrown if the cursor is at the end of the input. + */ char consume() { final char c = peek(); position++; return c; } + /** + * Consumes the next character, requiring it to be {@code c}. + * + * @param c The expected character. + * @throws IllegalArgumentException Thrown if the next character is not {@code c}. + */ void expect(char c) { final char actual = consume(); if (actual != c) { @@ -83,6 +100,11 @@ void requireEnd(String message) { } } + /** + * {@return the JSON string starting at the cursor, with escapes decoded} + * + * @throws IllegalArgumentException Thrown if the string is unterminated or has a bad escape. + */ String parseString() { expect('"'); final StringBuilder value = new StringBuilder(); @@ -102,6 +124,7 @@ String parseString() { } } + /** {@return the character named by the escape sequence following a backslash} */ private char parseEscape() { if (position >= text.length()) { throw malformed("Unterminated escape sequence"); @@ -121,6 +144,7 @@ private char parseEscape() { }; } + /** {@return the character named by a {@code \\uXXXX} escape} */ private char parseUnicodeEscape() { if (position + 4 > text.length()) { throw malformed("Truncated \\u escape sequence"); @@ -140,8 +164,10 @@ private char parseUnicodeEscape() { return (char) value; } - // Skips one number, holding it to the JSON grammar (optional minus, digits, optional - // fraction, optional signed exponent) so malformed input fails loud even in skipped fields. + /** + * Skips one JSON number, holding it to the grammar (optional minus, digits, optional fraction, + * optional signed exponent) so malformed input fails loud even in a skipped field. + */ private void skipNumber() { if (peek() == '-') { position++; @@ -177,6 +203,11 @@ private void skipNumber() { } } + /** + * {@return the integer starting at the cursor, parsed as a {@code long}} + * + * @throws IllegalArgumentException Thrown if no integer is present or it overflows a long. + */ long parseLong() { final int start = position; if (peek() == '-') { @@ -195,8 +226,10 @@ long parseLong() { } } - // Skips one JSON value of any type (string, number, array, object, true/false/null); used - // for fields a reader does not care about, so unknown additions never break it. + /** + * Skips one JSON value of any type (string, number, array, object, true/false/null), so a + * reader tolerates fields it does not care about. + */ void skipValue() { skipWhitespace(); final char c = peek(); @@ -252,6 +285,11 @@ void skipValue() { } } + /** + * {@return an exception naming the input and the cursor offset} + * + * @param message What was wrong at the cursor. + */ IllegalArgumentException malformed(String message) { return new IllegalArgumentException( "Malformed " + inputName + " at offset " + position + ": " + message); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 36c048b160..f8dd264b27 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -34,22 +34,14 @@ /** * Reads a safetensors file: an 8-byte * little-endian header length, a JSON header describing each tensor's dtype, shape, and byte - * range, followed by the raw tensor bytes. Deliberately not a general tensor-format library: - * only the {@code F32} decode path {@link #readFloat32(String)} needs is implemented, since - * that is what a distilled static-embedding table stores. + * range, followed by the raw tensor bytes. Only the {@code F32} decode path + * {@link #readFloat32(String)} needs is implemented. * - *

Security. Unlike PyTorch's pickle-based checkpoint format, safetensors carries no - * executable content: the header is data-only JSON and the body is raw tensor bytes, so loading - * one cannot execute arbitrary code. No hardening beyond ordinary malformed-input handling is - * needed.

- * - *

Only the header is read eagerly; tensor data is streamed straight into the caller's array - * with positional reads when requested, so the file size is not limited by Java's int-indexed - * arrays. The remaining ceiling is per tensor, not per file: one decoded {@code float[]} holds - * at most {@link Integer#MAX_VALUE} - 8 elements, and {@link #readFloat32(String)} checks that - * explicitly. The file must stay in place and unchanged between {@link #read(Path)} and later - * {@link #readFloat32(String)} calls; a file truncated in between fails loud rather than - * returning partial data.

+ *

Only the header is read eagerly; tensor data is streamed into a fresh array with positional + * reads on request, so a decoded {@code float[]} is capped at {@link Integer#MAX_VALUE} - 8 + * elements. The file must stay in place and unchanged between {@link #read(Path)} and a later + * {@link #readFloat32(String)} call; a file truncated in between fails loud rather than returning + * partial data.

* *

Instances are immutable and safe for concurrent use: every {@link #readFloat32(String)} * call opens its own channel and decodes into a fresh array the caller owns.

@@ -194,10 +186,6 @@ public float[] readFloat32(String name) throws IOException { + " F32 elements but its data range is " + byteLength + " bytes"); } final float[] values = new float[(int) elementCount]; - // When the build baseline reaches JDK 22+, this loop can become a single MemorySegment.copy - // out of a FileChannel.map'd segment (long-indexed, deterministic unmap via Arena); on the - // JDK 21 baseline java.lang.foreign is still a preview API, so positional reads are the - // portable way past the 2 GB byte[]/ByteBuffer ceiling. try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { final ByteBuffer chunk = ByteBuffer.allocate((int) Math.min(READ_CHUNK_BYTES, byteLength)) .order(ByteOrder.LITTLE_ENDIAN); @@ -220,9 +208,17 @@ public float[] readFloat32(String name) throws IOException { } } - // Fills the buffer with bytes starting at the given file position; fails loud if the file - // ends first, which can only happen when the file shrank after read(Path) validated ranges - // against its length. + /** + * Fills the buffer with bytes starting at the given file position. + * + * @param channel The open channel to read from. + * @param buffer The buffer to fill. + * @param position The starting file position. + * @param file The file, for error messages. + * @throws IOException Thrown if reading fails. + * @throws IllegalStateException Thrown if the file ends before the buffer is full, which can + * only happen when the file shrank after {@link #read(Path)} validated its ranges. + */ private static void readFully(FileChannel channel, ByteBuffer buffer, long position, Path file) throws IOException { while (buffer.hasRemaining()) { @@ -236,10 +232,9 @@ private static void readFully(FileChannel channel, ByteBuffer buffer, long posit } /** - * Finds the single 2-dimensional {@code F32} tensor in this file, the shape a static - * embedding table's weight matrix takes (vocabulary size by hidden dimension). Deliberately - * strict rather than guessing a name convention: distillation tools do not agree on one, and a - * wrong guess would silently load the wrong tensor. + * Finds the single 2-dimensional {@code F32} tensor in this file, the shape a static embedding + * table's weight matrix takes (vocabulary size by hidden dimension). Strict rather than guessing + * a name convention, so a wrong guess cannot silently load the wrong tensor. * * @return The name of the single 2-D F32 tensor. * @throws IllegalArgumentException Thrown if the file has zero or more than one 2-D F32 diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java index 41a54cb905..a2c1954b66 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java @@ -22,13 +22,9 @@ import java.util.Map; /** - * A cursor parser for the JSON header of a safetensors file. Purpose-built for the header's - * fixed, shallow shape (a flat object of tensor name to a {@code dtype}/{@code shape}/ - * {@code data_offsets} record, plus an optional {@code __metadata__} string map), not a - * general-purpose JSON parser: no floating-point numbers, no arbitrary nesting depth, no - * comments. This is the same discipline used by every other data-file cursor parser in the - * project (no regular expressions, fail loud on malformed input); the scanning primitives are - * shared with {@link FlatJsonFields} through {@link JsonCursor}. + * A cursor parser for the JSON header of a safetensors file: a flat object of tensor name to a + * {@code dtype}/{@code shape}/{@code data_offsets} record, plus an optional {@code __metadata__} + * string map. Not a general-purpose JSON parser; it fails loud on anything outside that shape. */ final class SafetensorsHeaderParser { @@ -57,6 +53,7 @@ static Result parse(String headerJson) { return parser.parseTop(); } + /** {@return the parsed header: its tensors in header order and the {@code __metadata__} map} */ private Result parseTop() { final List tensors = new ArrayList<>(); Map metadata = Map.of(); @@ -93,12 +90,19 @@ private Result parseTop() { return new Result(tensors, metadata); } - // Trailing whitespace is legal (writers space-pad the header to align the data section), but - // any other trailing content means the declared header length and the JSON disagree. + /** + * Requires the rest of the header to be whitespace only. Trailing whitespace is legal (writers + * space-pad the header to align the data section); other trailing content is a length mismatch. + */ private void requireEnd() { cursor.requireEnd("Trailing content after the header object"); } + /** + * {@return one tensor's metadata, parsed from its header record} + * + * @param name The tensor's name, the key it was declared under. + */ private TensorInfo parseTensorInfo(String name) { cursor.expect('{'); String dtype = null; @@ -145,6 +149,7 @@ private TensorInfo parseTensorInfo(String name) { + "and data_offsets"); } + /** {@return a JSON object of string values, used for the {@code __metadata__} map} */ private Map parseStringMap() { final Map map = new LinkedHashMap<>(); cursor.expect('{'); @@ -172,6 +177,11 @@ private Map parseStringMap() { } } + /** + * {@return a JSON array of non-negative integers as an {@code int[]}} + * + * @throws IllegalArgumentException Thrown if any element is outside the {@code int} range. + */ private int[] parseIntArray() { final long[] longs = parseLongArray(); final int[] ints = new int[longs.length]; @@ -184,6 +194,7 @@ private int[] parseIntArray() { return ints; } + /** {@return a JSON array of integers as a {@code long[]}} */ private long[] parseLongArray() { cursor.expect('['); cursor.skipWhitespace(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 1446334947..a3c45ab279 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -30,28 +30,16 @@ /** * A static (non-contextual) sentence embedding model: a per-token vector table plus WordPiece - * tokenization, the pure-JVM word2vec/GloVe successor described in the design doc this module - * implements. Distilled tables in this shape (Model2Vec and compatible releases) carry a modern - * sentence-transformer's semantics in a flat lookup table, so embedding a sentence is tokenize, - * gather, (optionally) weight, mean-pool, and (optionally) normalize: no model forward pass, no - * GPU, no native runtime. + * tokenization. Embedding a sentence is tokenize, gather each token's row, optionally weight, + * mean-pool, and optionally L2-normalize; there is no model forward pass. It loads distilled + * tables in the Model2Vec release layout: a {@code vocab.txt} and a {@code model.safetensors} + * holding one 2-D {@code F32} matrix, with an optional per-token {@code weights} tensor. * - *

The pooling formula matches the reference Model2Vec implementations exactly (verified - * against MinishLab's Rust {@code model2vec-rs}, not assumed): {@code [CLS]}/{@code [SEP]} are - * never added to the pool (this class tokenizes for lookup, not for a transformer), unknown - * tokens are dropped rather than contributing a meaningless vector, each remaining token's - * vector is multiplied by its optional per-token weight, the sum is divided by the plain count - * of pooled tokens (not the sum of weights), and if the model calls for normalization the - * pooled vector is L2-normalized with an epsilon floor so a token-less input yields a zero - * vector rather than a division by zero.

+ *

{@code [CLS]} and {@code [SEP]} are never pooled and unknown tokens are dropped; the sum is + * divided by the count of pooled tokens, not the sum of weights. A text with no in-vocabulary + * tokens yields a zero vector.

* - *

Thread safety. Instances are immutable and safe for concurrent use after - * construction: every field is final, the loaded arrays are never exposed or mutated, and the - * tokenizer chain holds no per-call state. The one piece of global mutable state in that chain, - * the {@code keepNewLines} flag on the {@code WhitespaceTokenizer.INSTANCE} singleton that - * {@link WordpieceTokenizer} splits with, cannot affect results here: BERT basic tokenization - * has already replaced every whitespace character, line breaks included, with plain spaces - * before that split runs, so the flag's only behavioral branch never triggers on this input.

+ *

Instances are immutable and safe for concurrent use after construction.

*/ @ThreadSafe public final class StaticEmbeddingModel implements TextEmbedder { @@ -94,8 +82,8 @@ public enum Normalization { private final WordpiecePipeline tokenizer; private final boolean normalize; private final String unknownToken; - // Per-row L2 norms and the special-token mask are constants of the model, precomputed at - // load time so the nearest-neighbor scan does no per-row square-root or string hashing. + // Per-row L2 norms and special-token mask, precomputed at load time so the neighbor scan + // does no per-row square root or string hashing. private final double[] rowNorms; private final boolean[] specialRows; @@ -116,18 +104,16 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, /** * Loads a static embedding model from a model directory, reading the tokenizer and pooling - * switches from the model's own configuration files instead of requiring the caller to know - * them: {@code normalize} from {@code config.json} and {@code do_lower_case} from - * {@code tokenizer_config.json}. The directory must contain {@code vocab.txt}, - * {@code model.safetensors}, {@code config.json}, and {@code tokenizer_config.json}, the - * layout Model2Vec-family releases publish (field names verified against published releases, - * not assumed). + * switches from the model's own configuration files: {@code normalize} from {@code config.json} + * and {@code do_lower_case} from {@code tokenizer_config.json}. The directory must contain + * {@code vocab.txt}, {@code model.safetensors}, {@code config.json}, and + * {@code tokenizer_config.json}. * - *

A {@code strip_accents} that is absent or JSON {@code null} follows the BERT convention - * of stripping accents exactly when lower-casing, which is what the single lower-case switch - * of {@link #load(Path, Path, Casing, Normalization)} does. A model that explicitly sets - * {@code strip_accents} against its {@code do_lower_case} value cannot be represented by - * that switch, so it is rejected rather than silently mis-tokenized.

+ *

A {@code strip_accents} that explicitly disagrees with {@code do_lower_case} cannot be + * represented by the single lower-case switch of + * {@link #load(Path, Path, Casing, Normalization)} and is rejected rather than silently + * mis-tokenized; when absent or {@code null} it follows the BERT convention of stripping + * accents exactly when lower-casing.

* * @param modelDirectory The model directory. Must not be {@code null} and must be a * directory. @@ -135,8 +121,7 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, * @throws IllegalArgumentException Thrown if {@code modelDirectory} is {@code null} or not a * directory, a required file is missing, a configuration file is malformed or lacks its * field, the accent handling is not representable, or the vocabulary and the embedding - * matrix disagree; the message names the explicit overload as the fallback for - * differently laid-out models. + * matrix disagree. * @throws IOException Thrown if reading a file fails. */ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException { @@ -190,9 +175,7 @@ private static Path requiredFile(Path modelDirectory, String name) { /** * Loads a static embedding model from a BERT-style {@code vocab.txt} and a safetensors weight - * file, the file pair a Model2Vec-family distillation publishes. No model is bundled with this - * module: the caller points at files they downloaded (see the module's design doc for the - * license posture). + * file. No model is bundled with this module; the caller supplies the files. * * @param vocabularyFile The {@code vocab.txt} file: one token per line, line number is the * token's row id. Must not be {@code null} and must exist. @@ -203,12 +186,9 @@ private static Path requiredFile(Path modelDirectory, String name) { * scalar per vocabulary row, is used as a per-token pooling weight * when present. * @param casing Whether the tokenizer lower-cases and strips accents - * ({@link Casing#UNCASED}, matching the uncased BGE/BERT family this - * module targets) or preserves case ({@link Casing#CASED}), matching - * the base model's tokenizer configuration. + * ({@link Casing#UNCASED}) or preserves case ({@link Casing#CASED}). * @param normalization Whether {@link #embed(String)} L2-normalizes its result - * ({@link Normalization#L2}), matching the source model's - * {@code config.json} {@code normalize} field. + * ({@link Normalization#L2}) or not ({@link Normalization#NONE}). * @return The loaded model. * @throws IllegalArgumentException Thrown if an argument is {@code null}, a file is missing * or malformed, or the vocabulary size and the embedding matrix's row count disagree. @@ -279,11 +259,10 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } /** - * Embeds a piece of text. + * {@inheritDoc} + * + *

A text with no in-vocabulary tokens yields a zero vector.

* - * @param text The text to embed. Must not be {@code null}. - * @return The pooled embedding vector, of length {@link #dimension()}. A text with no - * in-vocabulary tokens yields a zero vector. * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. */ @Override @@ -306,8 +285,7 @@ public float[] embed(String text) { if (text == null) { throw new IllegalArgumentException("Text must not be null"); } - // The tokenizer always wraps its output in [CLS] ... [SEP]; neither belongs in the pool - // (this is table lookup, not transformer input), so the first and last tokens are skipped. + // The tokenizer wraps its output in [CLS] ... [SEP]; skip both, they are never pooled. final String[] tokens = tokenizer.tokenize(text); final float[] sum = new float[dimension]; int pooledCount = 0; @@ -352,7 +330,7 @@ public float[] embed(String text) { return sum; } - /** {@return the dimension of every vector this model produces} */ + /** {@inheritDoc} */ @Override public int dimension() { return dimension; @@ -364,8 +342,7 @@ public int vocabularySize() { } /** - * Cosine similarity between two pieces of text's pooled embeddings, the classic word2vec-era - * convenience this module exists to modernize. + * Cosine similarity between two pieces of text's pooled embeddings. * * @param text1 The first text. Must not be {@code null}. * @param text2 The second text. Must not be {@code null}. @@ -385,9 +362,7 @@ public double similarity(String text1, String text2) { /** * Finds the vocabulary tokens whose vectors are nearest a piece of text's pooled embedding, - * most similar first. A brute-force scan over the whole vocabulary; fine for the vocabulary - * sizes this module targets (tens of thousands of rows), not an approximate-nearest-neighbor - * index (a documented follow-up, not v1 scope). + * most similar first. This is a brute-force scan over the whole vocabulary. * * @param text The query text. Must not be {@code null}. * @param topK The maximum number of results. Must be at least 1. @@ -443,16 +418,25 @@ public List analogy(String a, String b, String c, int topK) { return nearestNeighbors(target, topK, excludedRows(a, b, c)); } + /** + * Requires {@code topK} to be at least 1. + * + * @param topK The requested result count. + * @throws IllegalArgumentException Thrown if {@code topK} is less than 1. + */ private static void requirePositive(int topK) { if (topK < 1) { throw new IllegalArgumentException("TopK must be at least 1, got " + topK); } } - // The vocabulary rows the given terms tokenize to, ascending and duplicate-free. Folding the - // terms through the model's own tokenizer (rather than comparing raw input strings against - // vocabulary tokens) is what makes the exclusion case- and accent-insensitive on uncased - // models, and it tolerates equal terms, which Set.of would reject as duplicates. + /** + * {@return the vocabulary rows the given terms tokenize to, ascending and duplicate-free} + * Folding the terms through the model's own tokenizer keeps the exclusion case- and + * accent-insensitive on uncased models. + * + * @param terms The terms to fold and exclude. + */ private int[] excludedRows(String... terms) { final SortedSet rows = new TreeSet<>(); for (final String term : terms) { @@ -476,8 +460,16 @@ private int[] excludedRows(String... terms) { return sorted; } - // The scan visits rows in ascending order and sortedExcludedRows is ascending, so exclusion - // is a single pointer that advances past each excluded row as the scan reaches it. + /** + * Scans the whole vocabulary for the rows nearest {@code query}, most similar first. + * + * @param query The query vector. + * @param topK The maximum number of neighbors to return. + * @param sortedExcludedRows Row ids to skip, in ascending order; the scan advances a single + * pointer through them as it visits rows in order. + * @return Up to {@code topK} neighbors, most similar first; empty when {@code query} has no + * direction. + */ private List nearestNeighbors(float[] query, int topK, int[] sortedExcludedRows) { final double queryNorm = norm(query); if (queryNorm < NORMALIZE_EPSILON) { @@ -501,8 +493,7 @@ private List nearestNeighbors(float[] query, int topK, int[] sortedExc continue; } final int base = row * dimension; - // Four accumulators because the JIT must not reorder floating-point additions and so - // cannot unroll this reduction itself; the split summation order is chosen deliberately. + // Four accumulators so the JIT can vectorize the dot product without reordering FP adds. double dot0 = 0; double dot1 = 0; double dot2 = 0; @@ -528,6 +519,12 @@ private List nearestNeighbors(float[] query, int topK, int[] sortedExc return List.of(ordered); } + /** + * {@return the cosine similarity of two vectors, or {@code 0} when either has no direction} + * + * @param a The first vector. + * @param b The second vector, of the same length as {@code a}. + */ private static double cosineSimilarity(float[] a, float[] b) { double dot = 0; double normASquared = 0; @@ -541,6 +538,11 @@ private static double cosineSimilarity(float[] a, float[] b) { return denominator < NORMALIZE_EPSILON ? 0.0 : dot / denominator; } + /** + * {@return the L2 norm of a vector} + * + * @param vector The vector to measure. + */ private static double norm(float[] vector) { double sumOfSquares = 0; for (final float value : vector) { @@ -552,9 +554,7 @@ private static double norm(float[] vector) { /** * A bounded selection of the {@code k} highest-similarity rows, kept as a min-heap over * primitive parallel arrays: the root is always the weakest kept candidate, so a full scan - * decides most rows with one comparison against it and the selection allocates nothing per - * row (the previous implementation materialized and fully sorted one record per vocabulary - * row per query). + * decides most rows with one comparison against it and allocates nothing per row. */ private static final class TopK { @@ -562,11 +562,20 @@ private static final class TopK { private final int[] rows; private int size; + /** + * @param capacity The maximum number of rows to keep. + */ TopK(int capacity) { this.similarities = new double[capacity]; this.rows = new int[capacity]; } + /** + * Offers a candidate row, keeping it only if it ranks among the top {@code capacity}. + * + * @param row The candidate row id. + * @param similarity The row's similarity to the query. + */ void offer(int row, double similarity) { if (size < similarities.length) { int i = size++; @@ -587,18 +596,22 @@ void offer(int row, double similarity) { } } + /** {@return the number of rows currently held} */ int size() { return size; } + /** {@return the row id of the weakest held candidate, the heap root} */ int minRow() { return rows[0]; } + /** {@return the similarity of the weakest held candidate, the heap root} */ double minSimilarity() { return similarities[0]; } + /** Removes the weakest held candidate, the heap root. */ void removeMin() { size--; similarities[0] = similarities[size]; @@ -606,6 +619,7 @@ void removeMin() { siftDown(); } + /** Restores the min-heap invariant from the root downward. */ private void siftDown() { int i = 0; while (true) { @@ -626,6 +640,12 @@ private void siftDown() { } } + /** + * Swaps two heap entries in both parallel arrays. + * + * @param i The first index. + * @param j The second index. + */ private void swap(int i, int j) { final double similarity = similarities[i]; similarities[i] = similarities[j]; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java index e07c7c212b..a098d75433 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java @@ -70,6 +70,7 @@ public long elementCount() { return count; } + /** {@inheritDoc} */ @Override public boolean equals(Object other) { return other instanceof TensorInfo that @@ -78,6 +79,7 @@ public boolean equals(Object other) { && dataOffsetBegin == that.dataOffsetBegin && dataOffsetEnd == that.dataOffsetEnd; } + /** {@inheritDoc} */ @Override public int hashCode() { int result = name.hashCode(); @@ -88,6 +90,7 @@ public int hashCode() { return result; } + /** {@inheritDoc} */ @Override public String toString() { return "TensorInfo[name=" + name + ", dtype=" + dtype + ", shape=" + Arrays.toString(shape) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java index ac96f205c5..8adffbac51 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java @@ -25,13 +25,9 @@ /** * The full BERT tokenization pipeline used for embedding-table lookup: basic tokenization - * (control removal, whitespace normalization, CJK isolation, optional lower casing with - * accent stripping, punctuation isolation) followed by {@link WordpieceTokenizer} with the - * BERT special tokens and the reference 100-character word limit. - * - *

This module-internal pipeline exists so the embedding hot path depends only on the - * stable wordpiece stage; it produces pieces without offset bookkeeping, which the lookup - * path does not need.

+ * (control removal, whitespace normalization, CJK isolation, optional lower casing with accent + * stripping, punctuation isolation) followed by {@link WordpieceTokenizer} with the BERT special + * tokens and the 100-character word limit. It produces pieces without offset bookkeeping. */ final class WordpiecePipeline { @@ -40,6 +36,10 @@ final class WordpiecePipeline { private final WordpieceTokenizer wordpieceTokenizer; private final boolean lowerCase; + /** + * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. + * @param lowerCase Whether basic tokenization lower-cases and strips accents. + */ WordpiecePipeline(Set vocabulary, boolean lowerCase) { Objects.requireNonNull(vocabulary, "vocabulary must not be null"); this.wordpieceTokenizer = new WordpieceTokenizer(vocabulary, @@ -48,10 +48,16 @@ final class WordpiecePipeline { this.lowerCase = lowerCase; } + /** + * {@return the wordpiece tokens of {@code text}, wrapped in {@code [CLS]} and {@code [SEP]}} + * + * @param text The text to tokenize. + */ String[] tokenize(String text) { return wordpieceTokenizer.tokenize(normalize(text)); } + /** {@return {@code text} after BERT basic tokenization} */ private String normalize(String text) { String normalized = cleanText(text); normalized = isolateCjkCharacters(normalized); @@ -61,6 +67,7 @@ private String normalize(String text) { return isolatePunctuation(normalized); } + /** {@return {@code text} with null, replacement, and control characters removed} */ private static String cleanText(String text) { final StringBuilder cleaned = new StringBuilder(text.length()); text.codePoints().forEach(codePoint -> { @@ -76,6 +83,7 @@ private static String cleanText(String text) { return cleaned.toString(); } + /** {@return {@code text} with each CJK character surrounded by spaces} */ private static String isolateCjkCharacters(String text) { final StringBuilder spaced = new StringBuilder(text.length()); text.codePoints().forEach(codePoint -> { @@ -88,6 +96,7 @@ private static String isolateCjkCharacters(String text) { return spaced.toString(); } + /** {@return {@code text} with combining accent marks removed} */ private static String stripAccents(String text) { final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); final StringBuilder stripped = new StringBuilder(decomposed.length()); @@ -99,6 +108,7 @@ private static String stripAccents(String text) { return stripped.toString(); } + /** {@return {@code text} with each punctuation character surrounded by spaces} */ private static String isolatePunctuation(String text) { final StringBuilder spaced = new StringBuilder(text.length()); text.codePoints().forEach(codePoint -> { @@ -111,6 +121,7 @@ private static String isolatePunctuation(String text) { return spaced.toString(); } + /** {@return whether the code point is a control character, treating tab/newline/return as not} */ private static boolean isControl(int codePoint) { if (codePoint == '\t' || codePoint == '\n' || codePoint == '\r') { return false; @@ -122,6 +133,7 @@ private static boolean isControl(int codePoint) { }; } + /** {@return whether the code point is whitespace for BERT basic tokenization} */ private static boolean isWhitespace(int codePoint) { if (codePoint == ' ' || codePoint == '\t' || codePoint == '\n' || codePoint == '\r') { return true; @@ -129,6 +141,7 @@ private static boolean isWhitespace(int codePoint) { return Character.getType(codePoint) == Character.SPACE_SEPARATOR; } + /** {@return whether the code point is punctuation for BERT basic tokenization} */ private static boolean isPunctuation(int codePoint) { if ((codePoint >= 33 && codePoint <= 47) || (codePoint >= 58 && codePoint <= 64) || (codePoint >= 91 && codePoint <= 96) || (codePoint >= 123 && codePoint <= 126)) { @@ -143,6 +156,7 @@ private static boolean isPunctuation(int codePoint) { }; } + /** {@return whether the code point is a CJK ideograph} */ private static boolean isCjk(int codePoint) { return (codePoint >= 0x4E00 && codePoint <= 0x9FFF) || (codePoint >= 0x3400 && codePoint <= 0x4DBF) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java index b17a0c5741..696beba923 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java @@ -28,11 +28,9 @@ import opennlp.tools.commons.ThreadSafe; /** - * A BERT-style {@code vocab.txt} vocabulary: one token per line, the line number (0-based) is - * the token's id. This is the same file format {@code bert-base-uncased} and the BGE family of - * models ship (the tokenizer {@code minishlab/potion-base-8M} was distilled from), and it is the - * row index into a static-embedding table's weight matrix: row {@code id} is that token's - * vector. + * A BERT-style {@code vocab.txt} vocabulary: one token per line, the line number (0-based) is the + * token's id. That id is the row index into a static-embedding table's weight matrix: row + * {@code id} holds that token's vector. * *

Immutable and safe for concurrent reads after construction.

*/ @@ -66,7 +64,14 @@ static WordpieceVocabulary read(Path file) throws IOException { return fromLines(Files.readAllLines(file), file.toString()); } - // Package-private so tests can build a vocabulary from in-memory lines without a temp file. + /** + * Builds a vocabulary from in-memory lines, the token order. + * + * @param lines The tokens, one per element; the index is the token's id. + * @param sourceName The source's name, for error messages. + * @return The parsed vocabulary. + * @throws IllegalArgumentException Thrown if a token appears more than once. + */ static WordpieceVocabulary fromLines(List lines, String sourceName) { final Map idByToken = new LinkedHashMap<>(lines.size() * 2); for (int id = 0; id < lines.size(); id++) { From 3f9bead2319d842870713b9259ed6a690ef8a5af Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 18:39:56 -0400 Subject: [PATCH 40/82] OPENNLP-1877: Rewrite the opennlp-embeddings README with diagrams and align the Dev Manual chapter --- opennlp-docs/src/docbkx/embeddings.xml | 13 +- .../opennlp-embeddings/README.md | 141 ++++++++++++++++-- 2 files changed, 138 insertions(+), 16 deletions(-) diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index abfe59c075..693269838c 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -36,6 +36,13 @@ contextual model remains the better choice when distinguishing word senses in context is the point of the task. + + OpenNLP supports contextual, ONNX-backed sentence vectors as well, in the + opennlp-dl module. Those are inherently more accurate; Model2Vec trades + some accuracy for a large speed gain. Both paths implement the same + TextEmbedder interface (in opennlp-api), so an application can + swap one for the other without changing its calling code. + No model is bundled with the module. Callers point it at a model directory they downloaded; the table's own license applies to the table. @@ -76,8 +83,10 @@ StaticEmbeddingModel model = StaticEmbeddingModel.load( Instances are immutable and safe for concurrent use, so one loaded model can serve every thread of an application. Texts with no in-vocabulary tokens embed to a zero - vector rather than raising an error, and similarity reports - 0 for them. + vector rather than raising an error (matching the reference implementation), and + similarity reports 0 for them. This is a rare edge case for + text in the model's language, since WordPiece backs off to subwords; it mostly happens + for empty input or text outside the vocabulary's coverage.
diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md index 5c64565f80..acb63a86a1 100644 --- a/opennlp-extensions/opennlp-embeddings/README.md +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -17,25 +17,96 @@ # OpenNLP Static Embeddings -This module produces sentence and word embedding vectors from a static (non-contextual) embedding table: a per-token vector matrix plus WordPiece tokenization, the modern successor to the word2vec and GloVe workflow. Distillation tools can compress a sentence-transformer into such a flat table (the Model2Vec family of releases is the primary target), and looking a sentence up in the table approximates the transformer's semantics at a small fraction of the cost: embedding a text is tokenize, gather, mean-pool, and normalize. No model forward pass, no GPU, no native runtime, pure JVM. +Embeddings have become an essential part of AI workloads. As such, OpenNLP introduces a pure-JVM approach to embeddings with a modern Model2Vec engine. -## When to use it +Turn text into embedding vectors from a static (non-contextual) table: a per-token vector matrix plus WordPiece tokenization. It is the modern successor to the word2vec and GloVe workflow. Distillation tools can compress a sentence-transformer into such a flat table (the Model2Vec family is the primary target), and looking a sentence up in the table approximates the transformer's semantics at a fraction of the cost. There is no model forward pass, no GPU, and no native runtime; it is pure JVM. -Use this module when embedding throughput and deployment simplicity matter more than the last few points of retrieval quality: semantic similarity and deduplication, candidate retrieval for a heavier reranker, clustering, or classification features. A contextual model remains the better choice when distinguishing word senses in context is the point of the task. +OpenNLP also supports ONNX models, which are inherently more accurate. Model2Vec sacrifices some accuracy for a large speed gain, and OpenNLP recognizes that trade-off, so both embedding methods are supported and share the same `TextEmbedder` seam. -## Usage +## Quickstart -A downloaded model directory containing `vocab.txt`, `model.safetensors`, `config.json`, and `tokenizer_config.json` (the layout published releases use) loads with one call; the tokenizer and pooling switches are read from the model's own configuration: +Point `load` at a downloaded model directory, then embed: ```java StaticEmbeddingModel model = StaticEmbeddingModel.load(Path.of("/path/to/model-directory")); -float[] vector = model.embed("The quick brown fox"); -double similarity = model.similarity("coffee", "espresso"); -List neighbors = model.mostSimilar("coffee", 5); -List analogy = model.analogy("man", "king", "woman", 1); +float[] vector = model.embed("The quick brown fox"); +double similarity = model.similarity("coffee", "espresso"); +List near = model.mostSimilar("coffee", 5); +``` + +The directory is the layout published releases use (`vocab.txt`, `model.safetensors`, `config.json`, `tokenizer_config.json`); the tokenizer and pooling switches are read from the model's own config. One loaded model is immutable and thread-safe, so it can serve every thread of an application. + +## When to use it + +Reach for this when embedding throughput and deployment simplicity matter more than the last few points of retrieval quality: semantic similarity, deduplication, candidate retrieval in front of a heavier reranker, clustering, or features for a classifier. A contextual model is still the better choice when the task depends on distinguishing word senses in context. + +## How it works + +A static embedding model is a vocabulary and a matrix: one row per token, each row a vector of the model's dimension. Embedding runs entirely as table lookups and arithmetic: + +```mermaid +flowchart LR + A["text"] --> B["WordPiece tokenize"] + B --> C["gather token rows
drop unknown, skip special"] + C --> D["weight + mean-pool"] + D --> E["L2 normalize"] + E --> F["float[] vector"] ``` +1. **Tokenize.** WordPiece splits the text into subword tokens using the model's own vocabulary and casing rule. Special tokens are marked and never contribute to the pooled vector. +2. **Gather.** Each in-vocabulary token contributes its row from the matrix. Unknown tokens are dropped. A text with no in-vocabulary tokens embeds to a zero vector rather than raising. +3. **Weight and pool.** Per-token weights (when the model carries them) multiply into the running sum, and the sum is divided by the plain token count. This mean-pool matches the reference implementation of the targeted model family exactly, verified against it rather than assumed. +4. **Normalize.** The pooled vector is L2-normalized by default so cosine similarity is a dot product. Normalization can be turned off for models that expect raw pooled vectors. + +Per-row L2 norms and the special-token mask are precomputed at load time, so the neighbor scan and similarity calls do not recompute them on every query. + +### Loading + +The one-argument `load` reads the model's own configuration to resolve the tokenizer and pooling switches, so callers do not restate them: + +```mermaid +flowchart TD + L["StaticEmbeddingModel.load(dir)"] --> CFG["read config.json,
tokenizer_config.json"] + CFG --> CAS["casing = do_lower_case"] + CFG --> NRM["normalization"] + L --> VOC["vocab.txt to WordpieceVocabulary"] + L --> MAT["model.safetensors to matrix"] + CAS --> M["immutable, thread-safe model"] + NRM --> M + VOC --> M + MAT --> M +``` + +The weights are read with a purpose-built **safetensors** reader. Unlike pickle-based checkpoint formats, safetensors carries no executable content, so loading a downloaded file cannot execute arbitrary code. Tensor data streams directly into the decoded array, so the file size is not bound by Java's int-indexed arrays; a single decoded tensor is capped at the maximum Java array length (about 2.1 billion float elements), checked explicitly. + +## Architecture + +```mermaid +flowchart TD + subgraph MODEL["StaticEmbeddingModel"] + WV["WordpieceVocabulary"] + WP["WordpiecePipeline"] + MX["embedding matrix"] + end + SHP["SafetensorsHeaderParser"] --> SF["SafetensorsFile"] + SF --> MX + MODEL -. implements .-> TE["TextEmbedder
(opennlp-api)"] + DL["SentenceVectorsDL
(opennlp-dl, ONNX)"] -. implements .-> TE +``` + +`TextEmbedder` is the shared seam: the static path here and the contextual ONNX path in `opennlp-dl` both implement it, so callers can swap one for the other without touching their code. + +## Performance + +A static table wins on speed and footprint because there is no model forward pass: the hot path is a vocabulary lookup, a handful of vector adds, and one normalization. The module ships a JMH benchmark (`StaticEmbeddingModelBenchmark`) that measures `embed()` and `mostSimilar()` throughput, so you can reproduce numbers on your own hardware and model. + +In our measurements on the potion-base-8M distilled table, the JVM path ran roughly an order of magnitude faster single-threaded than the model2vec Python reference on the same table, at around a fifth of the resident memory, with output vectors matching the reference within floating-point tolerance. Parity was established before any of the throughput work, so the speed is not bought with accuracy. Treat these as a starting expectation: results depend on the model, the text length distribution, and the hardware, so run the benchmark on the model you plan to use. + +## Usage + +### Loading a non-standard layout + For a model laid out differently, the explicit overload takes the two data files and the two model properties directly: ```java @@ -45,10 +116,52 @@ StaticEmbeddingModel model = StaticEmbeddingModel.load( StaticEmbeddingModel.Normalization.L2); // from the model's config ``` -Instances are immutable and safe for concurrent use, so one loaded model can serve every thread of an application. Texts with no in-vocabulary tokens embed to a zero vector rather than raising an error. +### Neighbors and analogies + +`Neighbor` is a small record of the token and its cosine similarity: + +```java +for (Neighbor n : model.mostSimilar("coffee", 5)) { + System.out.println(n.token() + " " + n.similarity()); +} + +List king = model.analogy("man", "king", "woman", 1); +``` + +### Retrieval + +Embed a small corpus once, then rank documents against a query by cosine similarity. Because the vectors are L2-normalized, cosine is a plain dot product: + +```java +StaticEmbeddingModel model = StaticEmbeddingModel.load(modelDir); + +List docs = List.of( + "How do I brew espresso at home?", + "The history of tea in East Asia", + "Best grinders for pour-over coffee"); + +float[][] docVectors = docs.stream().map(model::embed).toArray(float[][]::new); +float[] query = model.embed("home espresso machine"); + +IntStream.range(0, docs.size()) + .boxed() + .sorted(Comparator.comparingDouble(i -> -dot(query, docVectors[i]))) + .forEach(i -> System.out.println(docs.get(i))); +``` + +Here `dot` is any dot product over two float arrays. For a full RAG-style retriever, keep the document vectors in whatever index you already use and score queries the same way. This applies to most modern search engines, since they tend to decouple the HNSW lookups from the vectors you feed them. + +## Getting a model + +No model is bundled. Point the module at files you download, and the table's own license applies to the table. The Model2Vec distilled releases (for example potion-base-8M) publish the exact directory layout the one-argument `load` expects: download that release's `vocab.txt`, `model.safetensors`, `config.json`, and `tokenizer_config.json` into one directory and pass the directory to `load`. + +## Notes and limits + +- Instances are immutable and safe for concurrent use, so one loaded model serves every thread. +- Static tables do not disambiguate word senses in context. If the task turns on context, use a contextual model. +- Out-of-vocabulary-only input embeds to a zero vector, matching the reference implementation. This is a rare edge case for text in the model's language (WordPiece backs off to subwords), and mostly happens for empty input or text outside the vocabulary's coverage. Decide in your code whether a zero vector means "no signal" for your use case. -## Notes +## See also -- No model is bundled. Callers point the module at files they downloaded, and the table's own license applies to the table. -- Weights are read with a purpose-built safetensors reader. Unlike pickle-based checkpoint formats, safetensors carries no executable content, so loading a file cannot execute arbitrary code. Tensor data streams directly into the decoded array, so file size is not limited by Java's int-indexed arrays; one decoded tensor is capped at the maximum Java array length (about 2.1 billion float elements), checked explicitly. -- The pooling formula matches the reference implementations of the targeted model family exactly (verified against them, not assumed): special tokens never pool, unknown tokens are dropped, per-token weights multiply into the sum, and the sum divides by the plain token count. +- The Dev Manual chapter (`opennlp-docs/src/docbkx/embeddings.xml`) for the same material in the manual. +- `opennlp-dl` for the contextual, ONNX-backed sentence vector path, which shares the `TextEmbedder` interface with this module. From 99538d315d646279e8fcbe82de02e13d74f7e0e5 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 18:44:11 -0400 Subject: [PATCH 41/82] OPENNLP-1877: Document implementation-defined empty-input behavior on TextEmbedder and both impls --- .../src/main/java/opennlp/tools/embeddings/TextEmbedder.java | 5 +++++ .../src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java index e5f5781f85..f1e4142090 100644 --- a/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java +++ b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java @@ -37,6 +37,11 @@ public interface TextEmbedder { /** * Embeds a piece of text. * + *

Behavior for empty text, or text with no tokens the embedder recognizes, is + * implementation-defined: an implementation may return a zero vector, the vector of a special + * or fallback token, or something else, and should document its choice. Callers that need a + * uniform response should handle it themselves.

+ * * @param text The text to embed; must not be null. * @return The embedding vector, of length {@link #dimension()}. * @throws IllegalArgumentException Thrown if {@code text} is null. diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java index a935a798c7..0d41958406 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java @@ -147,7 +147,10 @@ public float[] getVectors(final String sentence) throws OrtException { /** * {@inheritDoc} * - *

Adapts {@link #getVectors(String)} to the {@link TextEmbedder} contract.

+ *

Adapts {@link #getVectors(String)} to the {@link TextEmbedder} contract. Empty or + * unrecognized input is still run through the model, which returns the vector for the + * wrapped {@code [CLS] ... [SEP]} sequence; it is not special-cased to a zero vector the way + * the static-table embedder is.

* * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. * @throws IllegalStateException Thrown if inference fails; the cause carries the From 110613cbc7aefaf1421e0fea1adb77357db8e40b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 19:14:44 -0400 Subject: [PATCH 42/82] OPENNLP-1877: Read F16 and BF16 safetensors tensors, not just F32 --- .../opennlp/embeddings/SafetensorsFile.java | 133 ++++++++++++++---- .../embeddings/StaticEmbeddingModel.java | 4 +- .../embeddings/SafetensorsFileTest.java | 44 ++++++ .../embeddings/SafetensorsTestFiles.java | 32 ++++- .../embeddings/StaticEmbeddingModelTest.java | 18 +++ 5 files changed, 194 insertions(+), 37 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index f8dd264b27..5d8e6d1509 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.ShortBuffer; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -34,16 +35,17 @@ /** * Reads a safetensors file: an 8-byte * little-endian header length, a JSON header describing each tensor's dtype, shape, and byte - * range, followed by the raw tensor bytes. Only the {@code F32} decode path - * {@link #readFloat32(String)} needs is implemented. + * range, followed by the raw tensor bytes. The floating-point decode path + * {@link #readFloats(String)} supports the {@code F32}, {@code F16} (IEEE half) and {@code BF16} + * (bfloat16) dtypes, widening the two 16-bit types to {@code float}. * *

Only the header is read eagerly; tensor data is streamed into a fresh array with positional * reads on request, so a decoded {@code float[]} is capped at {@link Integer#MAX_VALUE} - 8 * elements. The file must stay in place and unchanged between {@link #read(Path)} and a later - * {@link #readFloat32(String)} call; a file truncated in between fails loud rather than returning + * {@link #readFloats(String)} call; a file truncated in between fails loud rather than returning * partial data.

* - *

Instances are immutable and safe for concurrent use: every {@link #readFloat32(String)} + *

Instances are immutable and safe for concurrent use: every {@link #readFloats(String)} * call opens its own channel and decodes into a fresh array the caller owns.

*/ @ThreadSafe @@ -158,22 +160,23 @@ public TensorInfo tensorInfo(String name) { } /** - * Decodes a {@code F32} tensor's data, streaming it from the file. + * Decodes a floating-point tensor's data to {@code float[]}, streaming it from the file. + * Accepts the {@code F32}, {@code F16} (IEEE half) and {@code BF16} (bfloat16) dtypes; the two + * 16-bit types are widened to {@code float} as they are read. {@code F16} is model2vec's + * default output dtype, so this is the common case for downloaded distilled tables. * * @param name The tensor's name. Must not be {@code null}. * @return The tensor's elements in row-major (shape outermost-first) order. * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, not a tensor in - * this file, not declared with dtype {@code F32}, or larger than a Java array can hold. + * this file, not a supported float dtype ({@code F32}, {@code F16}, {@code BF16}), or + * larger than a Java array can hold. * @throws IllegalStateException Thrown if the file has been truncated since * {@link #read(Path)} validated the tensor's byte range. * @throws IOException Thrown if reading the file fails. */ - public float[] readFloat32(String name) throws IOException { + public float[] readFloats(String name) throws IOException { final TensorInfo info = tensorInfo(name); - if (!"F32".equals(info.dtype())) { - throw new IllegalArgumentException( - "Tensor '" + name + "' has dtype " + info.dtype() + ", not F32"); - } + final int elementBytes = floatElementBytes(info.dtype(), name); final long elementCount = info.elementCount(); if (elementCount < 0 || elementCount > MAX_ARRAY_LENGTH) { throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount @@ -181,11 +184,12 @@ public float[] readFloat32(String name) throws IOException { + "); decoding to a float[] is capped there"); } final long byteLength = info.dataOffsetEnd() - info.dataOffsetBegin(); - if (byteLength != elementCount * Float.BYTES) { - throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount - + " F32 elements but its data range is " + byteLength + " bytes"); + if (byteLength != elementCount * elementBytes) { + throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount + " " + + info.dtype() + " elements but its data range is " + byteLength + " bytes"); } final float[] values = new float[(int) elementCount]; + final String dtype = info.dtype(); try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { final ByteBuffer chunk = ByteBuffer.allocate((int) Math.min(READ_CHUNK_BYTES, byteLength)) .order(ByteOrder.LITTLE_ENDIAN); @@ -193,21 +197,92 @@ public float[] readFloat32(String name) throws IOException { int decoded = 0; while (decoded < values.length) { chunk.clear(); - final long remainingBytes = byteLength - (long) decoded * Float.BYTES; + final long remainingBytes = byteLength - (long) decoded * elementBytes; if (remainingBytes < chunk.capacity()) { chunk.limit((int) remainingBytes); } readFully(channel, chunk, position, file); chunk.flip(); - final int floats = chunk.remaining() / Float.BYTES; - chunk.asFloatBuffer().get(values, decoded, floats); - decoded += floats; - position += (long) floats * Float.BYTES; + final int count = chunk.remaining() / elementBytes; + decodeInto(chunk, dtype, values, decoded, count); + decoded += count; + position += (long) count * elementBytes; } return values; } } + /** + * Decodes an {@code F32} tensor, rejecting any other dtype. Use {@link #readFloats(String)} to + * also accept {@code F16} and {@code BF16}. + * + * @param name The tensor's name. Must not be {@code null}. + * @return The tensor's elements in row-major (shape outermost-first) order. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, not a tensor in + * this file, not declared with dtype {@code F32}, or larger than a Java array can hold. + * @throws IllegalStateException Thrown if the file has been truncated since {@link #read(Path)}. + * @throws IOException Thrown if reading the file fails. + */ + public float[] readFloat32(String name) throws IOException { + final TensorInfo info = tensorInfo(name); + if (!"F32".equals(info.dtype())) { + throw new IllegalArgumentException( + "Tensor '" + name + "' has dtype " + info.dtype() + ", not F32"); + } + return readFloats(name); + } + + /** + * Widens one chunk of raw tensor bytes into the output array according to its dtype. + * + * @param chunk The raw little-endian bytes, positioned at the first element to decode. + * @param dtype The tensor's dtype ({@code F32}, {@code F16}, or {@code BF16}). + * @param out The destination array. + * @param offset The index in {@code out} to write the first decoded element to. + * @param count The number of elements to decode from {@code chunk}. + */ + private static void decodeInto(ByteBuffer chunk, String dtype, float[] out, int offset, + int count) { + switch (dtype) { + case "F32" -> chunk.asFloatBuffer().get(out, offset, count); + case "F16" -> { + final ShortBuffer shorts = chunk.asShortBuffer(); + for (int i = 0; i < count; i++) { + out[offset + i] = Float.float16ToFloat(shorts.get()); + } + } + case "BF16" -> { + // bfloat16 is the high 16 bits of a float32: shift back up and reinterpret. + final ShortBuffer shorts = chunk.asShortBuffer(); + for (int i = 0; i < count; i++) { + out[offset + i] = Float.intBitsToFloat((shorts.get() & 0xFFFF) << 16); + } + } + default -> throw new IllegalArgumentException("Unsupported float dtype: " + dtype); + } + } + + /** + * {@return the number of bytes one element of {@code dtype} occupies} + * + * @param dtype The tensor dtype. + * @param tensorName The tensor's name, for the error message. + * @throws IllegalArgumentException if {@code dtype} is not a supported float type. + */ + private static int floatElementBytes(String dtype, String tensorName) { + return switch (dtype) { + case "F32" -> Float.BYTES; + case "F16", "BF16" -> Short.BYTES; + default -> throw new IllegalArgumentException("Tensor '" + tensorName + "' has dtype " + + dtype + ", not a supported float type (F32, F16, BF16)"); + }; + } + + /** {@return whether {@code dtype} is a float type this reader decodes} */ + private static boolean isFloatDtype(String dtype) { + return "F32".equals(dtype) || "F16".equals(dtype) || "BF16".equals(dtype); + } + /** * Fills the buffer with bytes starting at the given file position. * @@ -232,22 +307,23 @@ private static void readFully(FileChannel channel, ByteBuffer buffer, long posit } /** - * Finds the single 2-dimensional {@code F32} tensor in this file, the shape a static embedding - * table's weight matrix takes (vocabulary size by hidden dimension). Strict rather than guessing - * a name convention, so a wrong guess cannot silently load the wrong tensor. + * Finds the single 2-dimensional floating-point tensor in this file (dtype {@code F32}, + * {@code F16}, or {@code BF16}), the shape a static embedding table's weight matrix takes + * (vocabulary size by hidden dimension). Strict rather than guessing a name convention, so a + * wrong guess cannot silently load the wrong tensor. * - * @return The name of the single 2-D F32 tensor. - * @throws IllegalArgumentException Thrown if the file has zero or more than one 2-D F32 + * @return The name of the single 2-D float tensor. + * @throws IllegalArgumentException Thrown if the file has zero or more than one 2-D float * tensor; the message lists every candidate so the caller can pick explicitly with - * {@link #readFloat32(String)}. + * {@link #readFloats(String)}. */ public String singleMatrixTensorName() { String found = null; for (final TensorInfo info : tensorsByName.values()) { - if ("F32".equals(info.dtype()) && info.shape().length == 2) { + if (isFloatDtype(info.dtype()) && info.shape().length == 2) { if (found != null) { throw new IllegalArgumentException( - "More than one 2-D F32 tensor in this file; specify the name explicitly. " + "More than one 2-D float tensor in this file; specify the name explicitly. " + "Candidates: " + tensorsByName.keySet()); } found = info.name(); @@ -255,7 +331,8 @@ public String singleMatrixTensorName() { } if (found == null) { throw new IllegalArgumentException( - "No 2-D F32 tensor in this file. Available tensors: " + tensorsByName.keySet()); + "No 2-D float (F32/F16/BF16) tensor in this file. Available tensors: " + + tensorsByName.keySet()); } return found; } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index a3c45ab279..8e359fafbc 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -223,11 +223,11 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil + "belong to the same model"); } final int dimension = matrixInfo.shape()[1]; - final float[] embeddings = tensors.readFloat32(matrixName); + final float[] embeddings = tensors.readFloats(matrixName); float[] weights = null; if (tensors.tensorNames().contains(WEIGHTS_TENSOR_NAME)) { - weights = tensors.readFloat32(WEIGHTS_TENSOR_NAME); + weights = tensors.readFloats(WEIGHTS_TENSOR_NAME); if (weights.length != vocabulary.size()) { throw new IllegalArgumentException("Tensor '" + WEIGHTS_TENSOR_NAME + "' in " + safetensorsFile + " has " + weights.length + " elements but the vocabulary has " diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java index 1595fd2c26..46b6226457 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java @@ -333,4 +333,48 @@ void testTensorInfoElementCountOverflowFailsLoudly() { assertThrows(IllegalArgumentException.class, crafted::elementCount); assertTrue(e.getMessage().contains("overflows"), e.getMessage()); } + + @Test + void testReadsF16TensorWidenedToFloat(@TempDir Path dir) throws IOException { + // F16 is model2vec's default output dtype, so this is the common downloaded-model case. + final Path file = dir.resolve("f16.safetensors"); + final float[] expected = {1.0f, -2.0f, 0.5f, 3.5f}; // all exact in IEEE half + SafetensorsTestFiles.write(file, "F16", SafetensorsTestFiles.vector("w", expected)); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + assertEquals("F16", parsed.tensorInfo("w").dtype()); + assertArrayEquals(expected, parsed.readFloats("w"), 1e-3f); + } + + @Test + void testReadsBf16TensorWidenedToFloat(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("bf16.safetensors"); + final float[] expected = {1.0f, -2.0f, 0.5f, 100.0f}; // exact in bfloat16 + SafetensorsTestFiles.write(file, "BF16", SafetensorsTestFiles.vector("w", expected)); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + assertEquals("BF16", parsed.tensorInfo("w").dtype()); + assertArrayEquals(expected, parsed.readFloats("w"), 1e-3f); + } + + @Test + void testSingleMatrixTensorNameAcceptsF16(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("f16-matrix.safetensors"); + SafetensorsTestFiles.write(file, "F16", + SafetensorsTestFiles.matrix("embeddings", new float[][] {{1f, 2f}, {3f, 4f}})); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + assertEquals("embeddings", parsed.singleMatrixTensorName()); + } + + @Test + void testReadFloat32StrictlyRejectsF16(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("f16-strict.safetensors"); + SafetensorsTestFiles.write(file, "F16", SafetensorsTestFiles.vector("w", new float[] {1f, 2f})); + + final SafetensorsFile parsed = SafetensorsFile.read(file); + // readFloats accepts it; the strict readFloat32 must not. + assertArrayEquals(new float[] {1f, 2f}, parsed.readFloats("w"), 1e-3f); + assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("w")); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java index f590a4edb3..dd503b9aa4 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java @@ -53,27 +53,45 @@ static Tensor vector(String name, float[] values) { } /** - * Writes a safetensors file holding the given F32 tensors, header first, data in declaration - * order. + * Writes a safetensors file holding the given tensors as {@code F32}, header first, data in + * declaration order. */ static void write(Path file, Tensor... tensors) throws IOException { + write(file, "F32", tensors); + } + + /** + * Writes a safetensors file encoding each tensor value as {@code dtype}, one of {@code F32}, + * {@code F16} (IEEE half), or {@code BF16} (bfloat16). The {@link Tensor} values stay + * {@code float}; they are converted to the target dtype's bytes here. + */ + static void write(Path file, String dtype, Tensor... tensors) throws IOException { + final int elementBytes = switch (dtype) { + case "F32" -> Float.BYTES; + case "F16", "BF16" -> Short.BYTES; + default -> throw new IllegalArgumentException("unsupported test dtype: " + dtype); + }; final ByteArrayOutputStream data = new ByteArrayOutputStream(); final StringJoiner header = new StringJoiner(",", "{", "}"); int offset = 0; for (final Tensor tensor : tensors) { final ByteBuffer buffer = - ByteBuffer.allocate(tensor.values().length * Float.BYTES) - .order(ByteOrder.LITTLE_ENDIAN); + ByteBuffer.allocate(tensor.values().length * elementBytes).order(ByteOrder.LITTLE_ENDIAN); for (final float value : tensor.values()) { - buffer.putFloat(value); + switch (dtype) { + case "F32" -> buffer.putFloat(value); + case "F16" -> buffer.putShort(Float.floatToFloat16(value)); + case "BF16" -> buffer.putShort((short) (Float.floatToIntBits(value) >>> 16)); + default -> throw new IllegalArgumentException("unsupported test dtype: " + dtype); + } } data.writeBytes(buffer.array()); final StringJoiner shape = new StringJoiner(",", "[", "]"); for (final int dimension : tensor.shape()) { shape.add(Integer.toString(dimension)); } - final int end = offset + tensor.values().length * Float.BYTES; - header.add("\"" + tensor.name() + "\":{\"dtype\":\"F32\",\"shape\":" + shape + final int end = offset + tensor.values().length * elementBytes; + header.add("\"" + tensor.name() + "\":{\"dtype\":\"" + dtype + "\",\"shape\":" + shape + ",\"data_offsets\":[" + offset + "," + end + "]}"); offset = end; } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index 6ebc06a049..a579a047f0 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -74,6 +74,12 @@ private static Path writeSafetensors(Path dir, boolean withWeights) throws IOExc return file; } + private static Path writeSafetensorsF16(Path dir) throws IOException { + final Path file = dir.resolve("model.safetensors"); + SafetensorsTestFiles.write(file, "F16", SafetensorsTestFiles.matrix("embeddings", ROWS)); + return file; + } + @Test void testEmbedMeanPoolsWithoutWeights(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = @@ -86,6 +92,18 @@ void testEmbedMeanPoolsWithoutWeights(@TempDir Path dir) throws IOException { assertArrayEquals(new float[] {3.5f, 35f, 350f}, result, 1e-5f); } + @Test + void testLoadsAnF16EmbeddingMatrix(@TempDir Path dir) throws IOException { + // model2vec writes float16 by default, so the loader must accept it and widen to float. + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensorsF16(dir), + Casing.UNCASED, Normalization.NONE); + + assertEquals(DIMENSION, model.dimension()); + // (hello + world) / 2 = [3.5, 35, 350]; the row values are all exact in IEEE half. + assertArrayEquals(new float[] {3.5f, 35f, 350f}, model.embed("hello world"), 1e-2f); + } + @Test void testEmbedAppliesPerTokenWeightsButDividesByTokenCount(@TempDir Path dir) throws IOException { From 78a3fd95784c02a9e079c5cc45a77ba2369a0462 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 20:20:29 -0400 Subject: [PATCH 43/82] OPENNLP-1877: Support SentencePiece models through the SubwordTokenizer seam Static embedding tables distilled from SentencePiece teachers (bge-m3 and the XLM-RoBERTa family) now load and embed, multilingual vectors included. The module's own WordPiece pipeline is replaced by the SubwordTokenizer seam: WordpieceEncoder from opennlp-api for WordPiece models, the pure-JVM SentencePieceTokenizer from opennlp-subword for SentencePiece models. Matrix rows are resolved by piece string, never by tokenizer id, because the trained .model file and the matrix vocabulary routinely order and offset ids differently; a load-time sweep verifies every poolable piece has a row, so a wrong file pairing fails loud at load, not at query time. The SentencePiece row order comes from the Unigram model.vocab list of tokenizer.json (with added_tokens overlaid), read by a purpose-built parser in the package's existing JsonCursor style. The directory loader detects the tokenizer family from the files present. Verified for exact output parity against the reference implementation on a real distilled multilingual model, including CJK input. --- opennlp-docs/src/docbkx/embeddings.xml | 54 ++- .../opennlp-embeddings/README.md | 52 ++- opennlp-extensions/opennlp-embeddings/pom.xml | 5 + ...cabulary.java => EmbeddingVocabulary.java} | 52 ++- .../embeddings/StaticEmbeddingModel.java | 394 ++++++++++++++---- .../embeddings/TokenizerJsonVocab.java | 336 +++++++++++++++ .../opennlp/embeddings/WordpiecePipeline.java | 170 -------- ...Test.java => EmbeddingVocabularyTest.java} | 18 +- ...StaticEmbeddingModelSentencePieceTest.java | 328 +++++++++++++++ .../embeddings/StaticEmbeddingModelTest.java | 2 +- .../embeddings/TokenizerJsonVocabTest.java | 174 ++++++++ .../embeddings/WordpiecePipelineTest.java | 138 ------ .../opennlp/embeddings/tiny-unigram.model | Bin 0 -> 245202 bytes rat-excludes | 2 + 14 files changed, 1285 insertions(+), 440 deletions(-) rename opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/{WordpieceVocabulary.java => EmbeddingVocabulary.java} (64%) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java delete mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java rename opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/{WordpieceVocabularyTest.java => EmbeddingVocabularyTest.java} (84%) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java delete mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/resources/opennlp/embeddings/tiny-unigram.model diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 693269838c..49bb92bff2 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -22,12 +22,14 @@ The opennlp-embeddings extension module produces sentence and word embedding vectors from a static (non-contextual) embedding table: a per-token vector - matrix plus WordPiece tokenization. It is the modern successor to the word2vec and - GloVe workflow. Distillation tools can compress a sentence-transformer into such a - flat table (the Model2Vec family of releases is the primary target), and looking a - sentence up in the table approximates the transformer's semantics at a small fraction - of the cost: embedding a text is tokenize, gather, mean-pool, and normalize, with no - model forward pass, no GPU, and no native runtime. + matrix plus subword tokenization, WordPiece or SentencePiece. It is the modern + successor to the word2vec and GloVe workflow. Distillation tools can compress a + sentence-transformer into such a flat table (the Model2Vec family of releases is the + primary target), and looking a sentence up in the table approximates the transformer's + semantics at a small fraction of the cost: embedding a text is tokenize, gather, + mean-pool, and normalize, with no model forward pass, no GPU, and no native runtime. + SentencePiece support covers multilingual tables distilled from encoders of the + XLM-RoBERTa family, whose vectors embed different languages into the same space. Use it when embedding throughput and a pure-JVM deployment matter more than the last @@ -52,10 +54,15 @@
Embedding Text with the API - A model directory containing vocab.txt, model.safetensors, - config.json, and tokenizer_config.json (the layout published - model releases use) loads with a single call; the tokenizer and pooling switches are - read from the model's own configuration files: + A model directory loads with a single call, and the tokenizer family is detected from + the files present. A WordPiece model carries vocab.txt, + model.safetensors, config.json, and + tokenizer_config.json; a SentencePiece model carries a trained + .model file (sentencepiece.bpe.model, + spiece.model, or tokenizer.model) next to + tokenizer.json, model.safetensors, and + config.json. The tokenizer and pooling switches are read from the model's + own configuration files: neighbors = model.mostSimilar("coffee", 5); List analogy = model.analogy("man", "king", "woman", 1);]]> - For a model laid out differently, the explicit overload takes the two data files and - the two switches directly: whether the tokenizer lower-cases (and strips accents), - and whether embeddings are L2-normalized. Both are properties of the model, published - in its configuration. + For a model laid out differently, the explicit overloads take the data files and the + switches directly. The WordPiece overload takes whether the tokenizer lower-cases + (and strips accents) and whether embeddings are L2-normalized; both are properties of + the model, published in its configuration. The SentencePiece overload has no casing + switch, because the trained .model file carries the model's own text + normalizer. + + Matrix rows are resolved by piece string, never by tokenizer id, because the two + files of a SentencePiece model routinely order and offset their ids differently. A + poolable piece with no matrix row fails loud at load time. Distillation output + usually ships without the trained .model file; copy that one file from + the teacher model's own repository into the model directory, and the loader names + exactly this fix if the file is missing. + Instances are immutable and safe for concurrent use, so one loaded model can serve every thread of an application. Texts with no in-vocabulary tokens embed to a zero vector rather than raising an error (matching the reference implementation), and similarity reports 0 for them. This is a rare edge case for - text in the model's language, since WordPiece backs off to subwords; it mostly happens - for empty input or text outside the vocabulary's coverage. + text in the model's language, since subword tokenization backs off to smaller pieces; + it mostly happens for empty input or text outside the vocabulary's coverage.
diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md index acb63a86a1..969d2eb066 100644 --- a/opennlp-extensions/opennlp-embeddings/README.md +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -19,7 +19,7 @@ Embeddings have become an essential part of AI workloads. As such, OpenNLP introduces a pure-JVM approach to embeddings with a modern Model2Vec engine. -Turn text into embedding vectors from a static (non-contextual) table: a per-token vector matrix plus WordPiece tokenization. It is the modern successor to the word2vec and GloVe workflow. Distillation tools can compress a sentence-transformer into such a flat table (the Model2Vec family is the primary target), and looking a sentence up in the table approximates the transformer's semantics at a fraction of the cost. There is no model forward pass, no GPU, and no native runtime; it is pure JVM. +Turn text into embedding vectors from a static (non-contextual) table: a per-token vector matrix plus subword tokenization, WordPiece or SentencePiece. It is the modern successor to the word2vec and GloVe workflow. Distillation tools can compress a sentence-transformer into such a flat table (the Model2Vec family is the primary target), and looking a sentence up in the table approximates the transformer's semantics at a fraction of the cost. Because SentencePiece models are supported, this includes multilingual tables distilled from encoders like the XLM-RoBERTa family. There is no model forward pass, no GPU, and no native runtime; it is pure JVM. OpenNLP also supports ONNX models, which are inherently more accurate. Model2Vec sacrifices some accuracy for a large speed gain, and OpenNLP recognizes that trade-off, so both embedding methods are supported and share the same `TextEmbedder` seam. @@ -35,7 +35,13 @@ double similarity = model.similarity("coffee", "espresso"); List near = model.mostSimilar("coffee", 5); ``` -The directory is the layout published releases use (`vocab.txt`, `model.safetensors`, `config.json`, `tokenizer_config.json`); the tokenizer and pooling switches are read from the model's own config. One loaded model is immutable and thread-safe, so it can serve every thread of an application. +The directory is the layout published releases use, and `load` detects the tokenizer family from the files present. A WordPiece model carries `vocab.txt`, `model.safetensors`, `config.json`, and `tokenizer_config.json`. A SentencePiece model carries a trained `.model` file (`sentencepiece.bpe.model`, `spiece.model`, or `tokenizer.model`) next to `tokenizer.json`, `model.safetensors`, and `config.json`. In both cases the tokenizer and pooling switches are read from the model's own config. One loaded model is immutable and thread-safe, so it can serve every thread of an application. + +A multilingual SentencePiece table embeds different languages into the same space, so similarity works across them: + +```java +model.similarity("The weather is beautiful today", "今天天气很好"); // same meaning, high score +``` ## When to use it @@ -47,15 +53,15 @@ A static embedding model is a vocabulary and a matrix: one row per token, each r ```mermaid flowchart LR - A["text"] --> B["WordPiece tokenize"] - B --> C["gather token rows
drop unknown, skip special"] + A["text"] --> B["subword tokenize
(WordPiece or SentencePiece)"] + B --> C["gather piece rows by string
drop unknown, skip special"] C --> D["weight + mean-pool"] D --> E["L2 normalize"] E --> F["float[] vector"] ``` -1. **Tokenize.** WordPiece splits the text into subword tokens using the model's own vocabulary and casing rule. Special tokens are marked and never contribute to the pooled vector. -2. **Gather.** Each in-vocabulary token contributes its row from the matrix. Unknown tokens are dropped. A text with no in-vocabulary tokens embeds to a zero vector rather than raising. +1. **Tokenize.** The model's own subword tokenizer splits the text into pieces: WordPiece with the model's casing rule, or a trained SentencePiece model that carries its own text normalizer. Special pieces (the WordPiece `[CLS]`/`[SEP]`/`[UNK]` frame, a SentencePiece model's control and unknown pieces) never contribute to the pooled vector. +2. **Gather.** Each piece contributes its matrix row, found by the piece *string* rather than the tokenizer's numeric id. The two files of a SentencePiece model routinely order and offset their ids differently (the fairseq convention shifts them by one, and distillation tools reorder the vocabulary outright), so string lookup is what keeps the pairing robust; a poolable piece with no matrix row fails loud at load time, not at query time. Unknown pieces are dropped, and a text with no in-vocabulary pieces embeds to a zero vector rather than raising. 3. **Weight and pool.** Per-token weights (when the model carries them) multiply into the running sum, and the sum is divided by the plain token count. This mean-pool matches the reference implementation of the targeted model family exactly, verified against it rather than assumed. 4. **Normalize.** The pooled vector is L2-normalized by default so cosine similarity is a dot product. Normalization can be turned off for models that expect raw pooled vectors. @@ -67,14 +73,17 @@ The one-argument `load` reads the model's own configuration to resolve the token ```mermaid flowchart TD - L["StaticEmbeddingModel.load(dir)"] --> CFG["read config.json,
tokenizer_config.json"] - CFG --> CAS["casing = do_lower_case"] - CFG --> NRM["normalization"] - L --> VOC["vocab.txt to WordpieceVocabulary"] + L["StaticEmbeddingModel.load(dir)"] --> DET{"vocab.txt present?"} + DET -- "yes: WordPiece" --> WCFG["read config.json,
tokenizer_config.json"] + WCFG --> CAS["casing = do_lower_case"] + DET -- "no: SentencePiece" --> SPM["load the trained .model
(its own normalizer, no casing switch)"] + SPM --> TJ["tokenizer.json vocab
names the matrix rows"] + TJ --> COV["verify every poolable piece
has a matrix row"] + L --> NRM["normalization from config.json"] L --> MAT["model.safetensors to matrix"] CAS --> M["immutable, thread-safe model"] + COV --> M NRM --> M - VOC --> M MAT --> M ``` @@ -85,17 +94,19 @@ The weights are read with a purpose-built **safetensors** reader. Unlike pickle- ```mermaid flowchart TD subgraph MODEL["StaticEmbeddingModel"] - WV["WordpieceVocabulary"] - WP["WordpiecePipeline"] + EV["EmbeddingVocabulary
(piece string to matrix row)"] + ST["SubwordTokenizer"] MX["embedding matrix"] end + WE["WordpieceEncoder
(opennlp-api)"] -. one of .-> ST + SP["SentencePieceTokenizer
(opennlp-subword)"] -. one of .-> ST SHP["SafetensorsHeaderParser"] --> SF["SafetensorsFile"] SF --> MX MODEL -. implements .-> TE["TextEmbedder
(opennlp-api)"] DL["SentenceVectorsDL
(opennlp-dl, ONNX)"] -. implements .-> TE ``` -`TextEmbedder` is the shared seam: the static path here and the contextual ONNX path in `opennlp-dl` both implement it, so callers can swap one for the other without touching their code. +Two seams keep the module small. `SubwordTokenizer` is the tokenization seam: the WordPiece encoder from `opennlp-api` and the pure-JVM SentencePiece implementation from `opennlp-subword` both produce the same piece stream, so the pooling code has exactly one path. `TextEmbedder` is the embedding seam: the static path here and the contextual ONNX path in `opennlp-dl` both implement it, so callers can swap one for the other without touching their code. ## Performance @@ -107,7 +118,7 @@ In our measurements on the potion-base-8M distilled table, the JVM path ran roug ### Loading a non-standard layout -For a model laid out differently, the explicit overload takes the two data files and the two model properties directly: +For a model laid out differently, the explicit overloads take the data files and the model properties directly. WordPiece: ```java StaticEmbeddingModel model = StaticEmbeddingModel.load( @@ -116,6 +127,15 @@ StaticEmbeddingModel model = StaticEmbeddingModel.load( StaticEmbeddingModel.Normalization.L2); // from the model's config ``` +SentencePiece (no casing switch, because the `.model` file carries the model's own text normalizer): + +```java +StaticEmbeddingModel model = StaticEmbeddingModel.loadSentencePiece( + Path.of("sentencepiece.bpe.model"), Path.of("tokenizer.json"), + Path.of("model.safetensors"), + StaticEmbeddingModel.Normalization.L2); +``` + ### Neighbors and analogies `Neighbor` is a small record of the token and its cosine similarity: @@ -155,6 +175,8 @@ Here `dot` is any dot product over two float arrays. For a full RAG-style retrie No model is bundled. Point the module at files you download, and the table's own license applies to the table. The Model2Vec distilled releases (for example potion-base-8M) publish the exact directory layout the one-argument `load` expects: download that release's `vocab.txt`, `model.safetensors`, `config.json`, and `tokenizer_config.json` into one directory and pass the directory to `load`. +For a multilingual SentencePiece table (for example one distilled from a bge-m3 or XLM-RoBERTa teacher), the distillation output ships `tokenizer.json`, `model.safetensors`, and `config.json` but usually not the trained SentencePiece `.model` file; copy that one file from the teacher model's own repository (it is named `sentencepiece.bpe.model` there) into the same directory. The loader tells you exactly this if the file is missing. + ## Notes and limits - Instances are immutable and safe for concurrent use, so one loaded model serves every thread. diff --git a/opennlp-extensions/opennlp-embeddings/pom.xml b/opennlp-extensions/opennlp-embeddings/pom.xml index 8ff56634d9..94d9907352 100644 --- a/opennlp-extensions/opennlp-embeddings/pom.xml +++ b/opennlp-extensions/opennlp-embeddings/pom.xml @@ -42,6 +42,11 @@ opennlp-runtime + + org.apache.opennlp + opennlp-subword + + org.junit.jupiter junit-jupiter-api diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java similarity index 64% rename from opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java rename to opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java index 696beba923..703e8c29ee 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpieceVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java @@ -28,25 +28,28 @@ import opennlp.tools.commons.ThreadSafe; /** - * A BERT-style {@code vocab.txt} vocabulary: one token per line, the line number (0-based) is the - * token's id. That id is the row index into a static-embedding table's weight matrix: row - * {@code id} holds that token's vector. + * The row table of a static embedding matrix: piece string to row index and back. Row {@code id} + * of the matrix holds the vector of the piece at position {@code id} in this vocabulary. + * + *

Two layouts produce it: a BERT-style {@code vocab.txt} (one token per line, the line number + * is the row), and a {@code tokenizer.json} with a Unigram model (the {@code model.vocab} list + * order is the row order, with {@code added_tokens} overlaid).

* *

Immutable and safe for concurrent reads after construction.

*/ @ThreadSafe -final class WordpieceVocabulary { +final class EmbeddingVocabulary { private final Map idByToken; private final List tokenById; - private WordpieceVocabulary(Map idByToken, List tokenById) { + private EmbeddingVocabulary(Map idByToken, List tokenById) { this.idByToken = idByToken; this.tokenById = tokenById; } /** - * Reads a {@code vocab.txt} file. + * Reads a {@code vocab.txt} file: one token per line, the line number (0-based) is the row. * * @param file The vocabulary file. Must not be {@code null} and must exist. * @return The parsed vocabulary. @@ -54,7 +57,7 @@ private WordpieceVocabulary(Map idByToken, List tokenBy * contains a duplicate token. * @throws IOException Thrown if reading the file fails. */ - static WordpieceVocabulary read(Path file) throws IOException { + static EmbeddingVocabulary fromVocabTxt(Path file) throws IOException { if (file == null) { throw new IllegalArgumentException("File must not be null"); } @@ -64,32 +67,57 @@ static WordpieceVocabulary read(Path file) throws IOException { return fromLines(Files.readAllLines(file), file.toString()); } + /** + * Reads the Unigram vocabulary of a {@code tokenizer.json} file: the {@code model.vocab} list + * order is the row order, with {@code added_tokens} overlaid. + * + * @param file The {@code tokenizer.json} file. Must not be {@code null} and must exist. + * @return The parsed vocabulary. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null}, missing, or not a + * well-formed Unigram {@code tokenizer.json}, or a piece appears more than once. + * @throws IOException Thrown if reading the file fails. + */ + static EmbeddingVocabulary fromTokenizerJson(Path file) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); + } + return fromLines(TokenizerJsonVocab.rows(file), file.toString()); + } + /** * Builds a vocabulary from in-memory lines, the token order. * - * @param lines The tokens, one per element; the index is the token's id. + * @param lines The tokens, one per element; the index is the token's row. * @param sourceName The source's name, for error messages. * @return The parsed vocabulary. * @throws IllegalArgumentException Thrown if a token appears more than once. */ - static WordpieceVocabulary fromLines(List lines, String sourceName) { + static EmbeddingVocabulary fromLines(List lines, String sourceName) { final Map idByToken = new LinkedHashMap<>(lines.size() * 2); for (int id = 0; id < lines.size(); id++) { final String token = lines.get(id); if (idByToken.putIfAbsent(token, id) != null) { throw new IllegalArgumentException( "Vocabulary " + sourceName + " declares token '" + token - + "' more than once, at lines " + idByToken.get(token) + " and " + id); + + "' more than once, at rows " + idByToken.get(token) + " and " + id); } } - return new WordpieceVocabulary(Collections.unmodifiableMap(idByToken), List.copyOf(lines)); + return new EmbeddingVocabulary(Collections.unmodifiableMap(idByToken), List.copyOf(lines)); } - /** {@return every token in this vocabulary, suitable for a WordpieceTokenizer} */ + /** {@return every token in this vocabulary, without order} */ Set tokens() { return idByToken.keySet(); } + /** {@return every token in row order, suitable for an id-is-index tokenizer constructor} */ + List orderedTokens() { + return tokenById; + } + /** * Looks up a token's row id. Returns a primitive with a {@code -1} sentinel rather than an * {@code OptionalInt} because this sits on the per-token hot path of diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 8e359fafbc..37cab554c9 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -23,21 +23,34 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.function.IntPredicate; +import opennlp.subword.sentencepiece.SentencePieceTokenizer; import opennlp.tools.commons.ThreadSafe; import opennlp.tools.embeddings.TextEmbedder; +import opennlp.tools.tokenize.SubwordPiece; +import opennlp.tools.tokenize.SubwordTokenizer; +import opennlp.tools.tokenize.WordpieceEncoder; import opennlp.tools.tokenize.WordpieceTokenizer; /** - * A static (non-contextual) sentence embedding model: a per-token vector table plus WordPiece - * tokenization. Embedding a sentence is tokenize, gather each token's row, optionally weight, - * mean-pool, and optionally L2-normalize; there is no model forward pass. It loads distilled - * tables in the Model2Vec release layout: a {@code vocab.txt} and a {@code model.safetensors} - * holding one 2-D {@code F32} matrix, with an optional per-token {@code weights} tensor. + * A static (non-contextual) sentence embedding model: a per-token vector table plus subword + * tokenization. Embedding a sentence is tokenize, gather each piece's row, optionally weight, + * mean-pool, and optionally L2-normalize; there is no model forward pass. * - *

{@code [CLS]} and {@code [SEP]} are never pooled and unknown tokens are dropped; the sum is - * divided by the count of pooled tokens, not the sum of weights. A text with no in-vocabulary - * tokens yields a zero vector.

+ *

It loads distilled tables in the Model2Vec release layout for both tokenizer families: + * WordPiece models carry a {@code vocab.txt} whose line number is the matrix row, and + * SentencePiece models carry a Unigram {@code tokenizer.json} whose {@code model.vocab} list + * order is the row order, next to the trained SentencePiece {@code .model} file that performs + * the segmentation. In both cases the {@code model.safetensors} holds one 2-D float matrix, with + * an optional per-token {@code weights} tensor. Matrix rows are resolved by piece string, + * never by tokenizer id, so the two files may order or offset their ids differently without + * corrupting lookups; a piece the matrix does not carry fails loud at load time.

+ * + *

Special pieces (the WordPiece {@code [CLS]}/{@code [SEP]}/{@code [UNK]} frame, a + * SentencePiece model's control and unknown pieces) are never pooled; the sum is divided by the + * count of pooled pieces, not the sum of weights. A text with no in-vocabulary pieces yields a + * zero vector.

* *

Instances are immutable and safe for concurrent use after construction.

*/ @@ -70,58 +83,73 @@ public enum Normalization { private static final String SAFETENSORS_FILE_NAME = "model.safetensors"; private static final String CONFIG_FILE_NAME = "config.json"; private static final String TOKENIZER_CONFIG_FILE_NAME = "tokenizer_config.json"; + private static final String TOKENIZER_JSON_FILE_NAME = "tokenizer.json"; + // The file names SentencePiece models ship their trained .model under, by convention family. + private static final List SENTENCEPIECE_MODEL_FILE_NAMES = + List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); private static final int[] NO_EXCLUDED_ROWS = new int[0]; // Never meaningful as a "similar word" result. - private static final Set SPECIAL_TOKENS = Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, - WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN); + private static final Set WORDPIECE_SPECIAL_TOKENS = + Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, + WordpieceTokenizer.BERT_UNK_TOKEN); + private static final Set SENTENCEPIECE_SPECIAL_TOKENS = + Set.of("", "", "", "", ""); private final float[] embeddings; private final float[] weights; private final int dimension; - private final WordpieceVocabulary vocabulary; - private final WordpiecePipeline tokenizer; + private final EmbeddingVocabulary vocabulary; + private final SubwordTokenizer tokenizer; + // Tokenizer-id-space test for pieces that are never pooled: the WordPiece frame and unknown + // pieces, or a SentencePiece model's control and unknown pieces (whose piece string is the + // unmatched surface text, not a vocabulary entry). + private final IntPredicate skipPieceId; private final boolean normalize; - private final String unknownToken; // Per-row L2 norms and special-token mask, precomputed at load time so the neighbor scan // does no per-row square root or string hashing. private final double[] rowNorms; private final boolean[] specialRows; private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, - WordpieceVocabulary vocabulary, WordpiecePipeline tokenizer, - boolean normalize, String unknownToken, double[] rowNorms, + EmbeddingVocabulary vocabulary, SubwordTokenizer tokenizer, + IntPredicate skipPieceId, boolean normalize, double[] rowNorms, boolean[] specialRows) { this.embeddings = embeddings; this.weights = weights; this.dimension = dimension; this.vocabulary = vocabulary; this.tokenizer = tokenizer; + this.skipPieceId = skipPieceId; this.normalize = normalize; - this.unknownToken = unknownToken; this.rowNorms = rowNorms; this.specialRows = specialRows; } /** - * Loads a static embedding model from a model directory, reading the tokenizer and pooling - * switches from the model's own configuration files: {@code normalize} from {@code config.json} - * and {@code do_lower_case} from {@code tokenizer_config.json}. The directory must contain - * {@code vocab.txt}, {@code model.safetensors}, {@code config.json}, and - * {@code tokenizer_config.json}. + * Loads a static embedding model from a model directory, detecting the tokenizer family from + * the files present and reading the pooling switch ({@code normalize}) from the model's + * {@code config.json}. + * + *

A directory with a {@code vocab.txt} is a WordPiece model; its casing is read from + * {@code do_lower_case} in {@code tokenizer_config.json}. A {@code strip_accents} that + * explicitly disagrees with {@code do_lower_case} cannot be represented by the single + * lower-case switch of {@link #load(Path, Path, Casing, Normalization)} and is rejected rather + * than silently mis-tokenized; when absent or {@code null} it follows the BERT convention of + * stripping accents exactly when lower-casing. When both layouts are present, the + * {@code vocab.txt} wins.

* - *

A {@code strip_accents} that explicitly disagrees with {@code do_lower_case} cannot be - * represented by the single lower-case switch of - * {@link #load(Path, Path, Casing, Normalization)} and is rejected rather than silently - * mis-tokenized; when absent or {@code null} it follows the BERT convention of stripping - * accents exactly when lower-casing.

+ *

A directory with a trained SentencePiece file ({@code sentencepiece.bpe.model}, + * {@code spiece.model}, or {@code tokenizer.model}) next to a Unigram {@code tokenizer.json} + * is a SentencePiece model; the {@code .model} file carries its own text normalizer, so there + * is no casing switch to read.

* * @param modelDirectory The model directory. Must not be {@code null} and must be a * directory. * @return The loaded model. * @throws IllegalArgumentException Thrown if {@code modelDirectory} is {@code null} or not a - * directory, a required file is missing, a configuration file is malformed or lacks its - * field, the accent handling is not representable, or the vocabulary and the embedding - * matrix disagree. + * directory, neither layout's files are present, a configuration file is malformed or + * lacks its field, the accent handling is not representable, or the tokenizer and the + * embedding matrix disagree. * @throws IOException Thrown if reading a file fails. */ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException { @@ -132,16 +160,45 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException throw new IllegalArgumentException( "Model directory does not exist or is not a directory: " + modelDirectory); } - final Path vocabularyFile = requiredFile(modelDirectory, VOCABULARY_FILE_NAME); + final Path vocabularyFile = modelDirectory.resolve(VOCABULARY_FILE_NAME); + if (Files.isRegularFile(vocabularyFile)) { + return loadWordpieceDirectory(modelDirectory, vocabularyFile); + } + final Path sentencePieceModelFile = firstRegularFile(modelDirectory, + SENTENCEPIECE_MODEL_FILE_NAMES); + final Path tokenizerJsonFile = modelDirectory.resolve(TOKENIZER_JSON_FILE_NAME); + if (sentencePieceModelFile != null && Files.isRegularFile(tokenizerJsonFile)) { + return loadSentencePiece(sentencePieceModelFile, tokenizerJsonFile, + requiredFile(modelDirectory, SAFETENSORS_FILE_NAME), + requiredNormalize(requiredFile(modelDirectory, CONFIG_FILE_NAME))); + } + if (Files.isRegularFile(tokenizerJsonFile)) { + throw new IllegalArgumentException("Model directory " + modelDirectory + " has a " + + TOKENIZER_JSON_FILE_NAME + " but no trained SentencePiece file (" + + String.join(", ", SENTENCEPIECE_MODEL_FILE_NAMES) + "); copy the .model file " + + "from the model's base tokenizer next to it"); + } + throw new IllegalArgumentException("Model directory " + modelDirectory + " has neither a " + + VOCABULARY_FILE_NAME + " (WordPiece layout) nor a " + TOKENIZER_JSON_FILE_NAME + + " with a trained SentencePiece file (SentencePiece layout)"); + } + + /** + * Loads the WordPiece directory layout, reading the tokenizer and pooling switches from the + * model's own configuration files. + * + * @param modelDirectory The model directory. + * @param vocabularyFile The directory's {@code vocab.txt}. + * @return The loaded model. + * @throws IOException Thrown if reading a file fails. + */ + private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, + Path vocabularyFile) + throws IOException { final Path safetensorsFile = requiredFile(modelDirectory, SAFETENSORS_FILE_NAME); - final Path configFile = requiredFile(modelDirectory, CONFIG_FILE_NAME); final Path tokenizerConfigFile = requiredFile(modelDirectory, TOKENIZER_CONFIG_FILE_NAME); - final Boolean normalize = FlatJsonFields.topLevelBoolean(configFile, "normalize"); - if (normalize == null) { - throw new IllegalArgumentException(configFile + " has no boolean 'normalize' field; " - + "use load(vocabularyFile, safetensorsFile, casing, normalization) and choose " - + "explicitly"); - } + final Normalization normalization = + requiredNormalize(requiredFile(modelDirectory, CONFIG_FILE_NAME)); final Boolean lowerCase = FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "do_lower_case"); if (lowerCase == null) { @@ -159,28 +216,62 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException + "deliberately"); } return load(vocabularyFile, safetensorsFile, - lowerCase ? Casing.UNCASED : Casing.CASED, - normalize ? Normalization.L2 : Normalization.NONE); + lowerCase ? Casing.UNCASED : Casing.CASED, normalization); + } + + /** + * Reads the required {@code normalize} switch out of a model's {@code config.json}. + * + * @param configFile The {@code config.json} file. + * @return The corresponding {@link Normalization}. + * @throws IllegalArgumentException Thrown if the field is missing or not a boolean. + * @throws IOException Thrown if reading the file fails. + */ + private static Normalization requiredNormalize(Path configFile) throws IOException { + final Boolean normalize = FlatJsonFields.topLevelBoolean(configFile, "normalize"); + if (normalize == null) { + throw new IllegalArgumentException(configFile + " has no boolean 'normalize' field; " + + "use the explicit load overloads and choose the normalization deliberately"); + } + return normalize ? Normalization.L2 : Normalization.NONE; + } + + /** + * {@return the first of the given file names that exists as a regular file in the directory, + * or {@code null} when none does} + * + * @param directory The directory to look in. + * @param names The file names to try, in order. + */ + private static Path firstRegularFile(Path directory, List names) { + for (final String name : names) { + final Path file = directory.resolve(name); + if (Files.isRegularFile(file)) { + return file; + } + } + return null; } private static Path requiredFile(Path modelDirectory, String name) { final Path file = modelDirectory.resolve(name); if (!Files.isRegularFile(file)) { throw new IllegalArgumentException("Model directory " + modelDirectory + " has no " - + name + "; for a different layout, use load(vocabularyFile, safetensorsFile, " - + "casing, normalization)"); + + name + "; for a different layout, use the explicit load overloads"); } return file; } /** - * Loads a static embedding model from a BERT-style {@code vocab.txt} and a safetensors weight - * file. No model is bundled with this module; the caller supplies the files. + * Loads a WordPiece static embedding model from a BERT-style {@code vocab.txt} and a + * safetensors weight file. No model is bundled with this module; the caller supplies the + * files. * * @param vocabularyFile The {@code vocab.txt} file: one token per line, line number is the - * token's row id. Must not be {@code null} and must exist. + * token's row id. Must not be {@code null}, must exist, and must + * contain the {@code [CLS]}, {@code [SEP]}, and {@code [UNK]} tokens. * @param safetensorsFile The {@code model.safetensors} file. Must not be {@code null} and - * must exist, and must contain exactly one 2-D {@code F32} tensor + * must exist, and must contain exactly one 2-D float tensor * (the embedding matrix) whose row count matches the vocabulary size. * An optional 1-D {@code F32} tensor named {@code "weights"}, one * scalar per vocabulary row, is used as a per-token pooling weight @@ -209,15 +300,145 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil if (normalization == null) { throw new IllegalArgumentException("Normalization must not be null"); } - final boolean lowerCase = casing == Casing.UNCASED; - final boolean normalize = normalization == Normalization.L2; - final WordpieceVocabulary vocabulary = WordpieceVocabulary.read(vocabularyFile); - final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); + final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromVocabTxt(vocabularyFile); + final Matrix matrix = readMatrix(vocabulary, safetensorsFile, vocabularyFile.toString()); + final WordpieceEncoder tokenizer = + new WordpieceEncoder(vocabulary.orderedTokens(), casing == Casing.UNCASED); + // The encoder validated the frame tokens' presence, so these rows exist. + final int classificationId = vocabulary.id(WordpieceTokenizer.BERT_CLS_TOKEN); + final int separatorId = vocabulary.id(WordpieceTokenizer.BERT_SEP_TOKEN); + final int unknownId = vocabulary.id(WordpieceTokenizer.BERT_UNK_TOKEN); + final IntPredicate skipPieceId = + id -> id == classificationId || id == separatorId || id == unknownId; + return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), + vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, + rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size()), + specialRows(vocabulary, WORDPIECE_SPECIAL_TOKENS)); + } + /** + * Loads a SentencePiece static embedding model from a trained SentencePiece {@code .model} + * file, the Unigram {@code tokenizer.json} naming the matrix rows, and a safetensors weight + * file. No model is bundled with this module; the caller supplies the files. + * + *

The {@code .model} file carries the model's own text normalizer and segmentation state, + * so there is no casing switch. The two vocabulary files may order or offset their ids + * differently: matrix rows are resolved by piece string, and every piece the tokenizer can + * emit (except its control and unknown pieces, which are never pooled) must be present in the + * {@code tokenizer.json} vocabulary, verified once at load time.

+ * + * @param sentencePieceModelFile The trained SentencePiece {@code .model} file. Must not be + * {@code null} and must exist. + * @param tokenizerJsonFile The Unigram {@code tokenizer.json} file; its + * {@code model.vocab} list order is the matrix row order, with + * {@code added_tokens} overlaid. Must not be {@code null} and + * must exist. + * @param safetensorsFile The {@code model.safetensors} file. Must not be {@code null} + * and must exist, and must contain exactly one 2-D float tensor + * (the embedding matrix) whose row count matches the vocabulary + * size. An optional 1-D {@code F32} tensor named + * {@code "weights"}, one scalar per vocabulary row, is used as + * a per-token pooling weight when present. + * @param normalization Whether {@link #embed(String)} L2-normalizes its result + * ({@link Normalization#L2}) or not ({@link Normalization#NONE}). + * @return The loaded model. + * @throws IllegalArgumentException Thrown if an argument is {@code null}, a file is missing + * or malformed, the vocabulary size and the embedding matrix's row count disagree, or the + * tokenizer emits pieces the vocabulary does not map. + * @throws IOException Thrown if reading a file fails. + */ + public static StaticEmbeddingModel loadSentencePiece(Path sentencePieceModelFile, + Path tokenizerJsonFile, + Path safetensorsFile, + Normalization normalization) + throws IOException { + if (sentencePieceModelFile == null) { + throw new IllegalArgumentException("SentencePieceModelFile must not be null"); + } + if (tokenizerJsonFile == null) { + throw new IllegalArgumentException("TokenizerJsonFile must not be null"); + } + if (safetensorsFile == null) { + throw new IllegalArgumentException("SafetensorsFile must not be null"); + } + if (normalization == null) { + throw new IllegalArgumentException("Normalization must not be null"); + } + final EmbeddingVocabulary vocabulary = + EmbeddingVocabulary.fromTokenizerJson(tokenizerJsonFile); + final SentencePieceTokenizer tokenizer = + SentencePieceTokenizer.load(sentencePieceModelFile); + requireVocabularyCoverage(tokenizer, vocabulary, sentencePieceModelFile, tokenizerJsonFile); + final Matrix matrix = readMatrix(vocabulary, safetensorsFile, tokenizerJsonFile.toString()); + final IntPredicate skipPieceId = + id -> tokenizer.isUnknown(id) || tokenizer.isControl(id); + return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), + vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, + rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size()), + specialRows(vocabulary, SENTENCEPIECE_SPECIAL_TOKENS)); + } + + /** + * Verifies once at load time that every piece the tokenizer can emit maps to a matrix row, so + * embedding never meets an unmapped piece. Control and unknown pieces are exempt: they are + * never pooled, and a distillation legitimately drops them from the matrix. + * + * @param tokenizer The loaded SentencePiece tokenizer. + * @param vocabulary The matrix row vocabulary. + * @param sentencePieceModelFile The tokenizer's source file, for error messages. + * @param tokenizerJsonFile The vocabulary's source file, for error messages. + * @throws IllegalArgumentException Thrown if a poolable piece has no matrix row. + */ + private static void requireVocabularyCoverage(SentencePieceTokenizer tokenizer, + EmbeddingVocabulary vocabulary, + Path sentencePieceModelFile, + Path tokenizerJsonFile) { + int missing = 0; + final StringBuilder samples = new StringBuilder(); + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + if (tokenizer.isUnknown(id) || tokenizer.isControl(id)) { + continue; + } + if (vocabulary.id(tokenizer.idToPiece(id)) < 0) { + if (missing < 5) { + if (missing > 0) { + samples.append(", "); + } + samples.append('\'').append(tokenizer.idToPiece(id)).append('\''); + } + missing++; + } + } + if (missing > 0) { + throw new IllegalArgumentException(sentencePieceModelFile + " defines " + missing + + " pieces that " + tokenizerJsonFile + " does not map to a matrix row (first: " + + samples + "); these files do not belong to the same model"); + } + } + + /** The embedding matrix and its optional per-token weights, as read from a safetensors file. */ + private record Matrix(float[] embeddings, float[] weights, int dimension) { + } + + /** + * Reads the embedding matrix and the optional {@code weights} tensor, holding both to the + * vocabulary's size. + * + * @param vocabulary The matrix row vocabulary. + * @param safetensorsFile The safetensors file to read. + * @param vocabularySourceName The vocabulary's source, for error messages. + * @return The matrix, its optional weights, and its dimension. + * @throws IllegalArgumentException Thrown if the matrix's row count or the weights tensor's + * length disagrees with the vocabulary size. + * @throws IOException Thrown if reading the file fails. + */ + private static Matrix readMatrix(EmbeddingVocabulary vocabulary, Path safetensorsFile, + String vocabularySourceName) throws IOException { + final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); final String matrixName = tensors.singleMatrixTensorName(); final TensorInfo matrixInfo = tensors.tensorInfo(matrixName); if (matrixInfo.shape()[0] != vocabulary.size()) { - throw new IllegalArgumentException("Vocabulary " + vocabularyFile + " has " + throw new IllegalArgumentException("Vocabulary " + vocabularySourceName + " has " + vocabulary.size() + " tokens but embedding matrix '" + matrixName + "' in " + safetensorsFile + " has " + matrixInfo.shape()[0] + " rows; these files do not " + "belong to the same model"); @@ -234,9 +455,19 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil + vocabulary.size() + " tokens"); } } + return new Matrix(embeddings, weights, dimension); + } - final double[] rowNorms = new double[vocabulary.size()]; - for (int row = 0; row < rowNorms.length; row++) { + /** + * {@return the L2 norm of every matrix row, precomputed for the neighbor scan} + * + * @param embeddings The flat row-major matrix. + * @param dimension The row width. + * @param rowCount The number of rows. + */ + private static double[] rowNorms(float[] embeddings, int dimension, int rowCount) { + final double[] rowNorms = new double[rowCount]; + for (int row = 0; row < rowCount; row++) { final int base = row * dimension; double sumOfSquares = 0; for (int d = 0; d < dimension; d++) { @@ -245,17 +476,26 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } rowNorms[row] = Math.sqrt(sumOfSquares); } + return rowNorms; + } + + /** + * {@return the mask of rows holding special tokens, excluded from neighbor results} + * + * @param vocabulary The matrix row vocabulary. + * @param specialTokens The special-token strings of the model's convention; tokens absent + * from the vocabulary are simply not marked. + */ + private static boolean[] specialRows(EmbeddingVocabulary vocabulary, + Set specialTokens) { final boolean[] specialRows = new boolean[vocabulary.size()]; - for (final String special : SPECIAL_TOKENS) { + for (final String special : specialTokens) { final int row = vocabulary.id(special); if (row >= 0) { specialRows[row] = true; } } - - final WordpiecePipeline tokenizer = new WordpiecePipeline(vocabulary.tokens(), lowerCase); - return new StaticEmbeddingModel(embeddings, weights, dimension, vocabulary, tokenizer, - normalize, WordpieceTokenizer.BERT_UNK_TOKEN, rowNorms, specialRows); + return specialRows; } /** @@ -285,20 +525,19 @@ public float[] embed(String text) { if (text == null) { throw new IllegalArgumentException("Text must not be null"); } - // The tokenizer wraps its output in [CLS] ... [SEP]; skip both, they are never pooled. - final String[] tokens = tokenizer.tokenize(text); + final List pieces = tokenizer.encode(text); final float[] sum = new float[dimension]; int pooledCount = 0; - for (int i = 1; i < tokens.length - 1; i++) { - final String token = tokens[i]; - if (unknownToken.equals(token)) { + for (int i = 0; i < pieces.size(); i++) { + final SubwordPiece piece = pieces.get(i); + if (skipPieceId.test(piece.id())) { continue; } - final int row = vocabulary.id(token); + final int row = vocabulary.id(piece.piece()); if (row < 0) { - throw new IllegalStateException("Tokenizer produced token '" + token - + "' that is not in its own vocabulary; this indicates a tokenizer/vocabulary " - + "construction bug, not an input problem"); + throw new IllegalStateException("Tokenizer produced piece '" + piece.piece() + + "' that has no matrix row; load-time validation admits no such piece, so this " + + "indicates a construction bug, not an input problem"); } final int base = row * dimension; if (weights == null) { @@ -366,9 +605,8 @@ public double similarity(String text1, String text2) { * * @param text The query text. Must not be {@code null}. * @param topK The maximum number of results. Must be at least 1. - * @return Up to {@code topK} neighbors, most similar first, excluding the special tokens - * ({@code [CLS]}, {@code [SEP]}, {@code [UNK]}); empty when {@code text} has no - * in-vocabulary tokens. + * @return Up to {@code topK} neighbors, most similar first, excluding the model's special + * tokens; empty when {@code text} has no in-vocabulary tokens. * @throws IllegalArgumentException Thrown if {@code text} is {@code null} or {@code topK} is * less than 1. */ @@ -389,10 +627,10 @@ public List mostSimilar(String text, int topK) { * @param b The second term. Must not be {@code null}. * @param c The third term. Must not be {@code null}. * @param topK The maximum number of results. Must be at least 1. - * @return Up to {@code topK} neighbors, most similar first, excluding the special tokens and - * every vocabulary token the three terms themselves tokenize to. The exclusion folds the - * terms exactly the way {@link #embed(String)} folds text, so on an uncased model a - * capitalized input excludes its lower-cased vocabulary row, and a multiword term + * @return Up to {@code topK} neighbors, most similar first, excluding the model's special + * tokens and every vocabulary token the three terms themselves tokenize to. The exclusion + * folds the terms exactly the way {@link #embed(String)} folds text, so on an uncased + * model a capitalized input excludes its lower-cased vocabulary row, and a multiword term * excludes each of its word pieces. * @throws IllegalArgumentException Thrown if {@code a}, {@code b}, or {@code c} is * {@code null}, or {@code topK} is less than 1. @@ -433,20 +671,18 @@ private static void requirePositive(int topK) { /** * {@return the vocabulary rows the given terms tokenize to, ascending and duplicate-free} * Folding the terms through the model's own tokenizer keeps the exclusion case- and - * accent-insensitive on uncased models. + * accent-insensitive on models that normalize. * * @param terms The terms to fold and exclude. */ private int[] excludedRows(String... terms) { final SortedSet rows = new TreeSet<>(); for (final String term : terms) { - final String[] tokens = tokenizer.tokenize(term); - for (int i = 1; i < tokens.length - 1; i++) { - final String token = tokens[i]; - if (unknownToken.equals(token)) { + for (final SubwordPiece piece : tokenizer.encode(term)) { + if (skipPieceId.test(piece.id())) { continue; } - final int row = vocabulary.id(token); + final int row = vocabulary.id(piece.piece()); if (row >= 0) { rows.add(row); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java new file mode 100644 index 0000000000..c4e5959204 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +/** + * Reads the row order of a static embedding matrix out of a {@code tokenizer.json} file with a + * Unigram model: the {@code model.vocab} list holds {@code [piece, score]} pairs whose index is + * the piece's id, and the {@code added_tokens} list overlays extra pieces (appended when their id + * is the next row, checked for agreement when it is an existing row). Only the vocabulary is + * read; every other section, including the tokenizer's normalizer and segmentation state, is + * skipped structurally. Like the package's other readers this is purpose-built on + * {@link JsonCursor}, not a general JSON library, and fails loud on anything outside the known + * shape. + */ +final class TokenizerJsonVocab { + + private TokenizerJsonVocab() { + } + + /** + * One entry of the {@code added_tokens} list. + * + * @param id The token's id, the matrix row it claims. + * @param content The token's string. + */ + private record AddedToken(long id, String content) { + } + + /** + * Reads the pieces of a Unigram {@code tokenizer.json} in row order. + * + * @param file The {@code tokenizer.json} file. Must not be {@code null} and must exist. + * @return The pieces; the index is the matrix row. + * @throws IllegalArgumentException Thrown if the file is not a well-formed + * {@code tokenizer.json}, its model is not Unigram, or an added token's id neither matches + * an existing row nor appends as the next one. + * @throws IOException Thrown if reading the file fails. + */ + static List rows(Path file) throws IOException { + final String json = Files.readString(file); + final JsonCursor cursor = new JsonCursor(json, file.getFileName().toString()); + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + + List vocab = null; + String modelType = null; + List addedTokens = List.of(); + boolean modelSeen = false; + boolean addedTokensSeen = false; + + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "model" -> { + if (modelSeen) { + throw cursor.malformed("Field 'model' appears more than once"); + } + modelSeen = true; + final ParsedModel model = parseModel(cursor); + vocab = model.vocab; + modelType = model.type; + } + case "added_tokens" -> { + if (addedTokensSeen) { + throw cursor.malformed("Field 'added_tokens' appears more than once"); + } + addedTokensSeen = true; + addedTokens = parseAddedTokens(cursor); + } + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a field, got '" + next + "'"); + } + } + cursor.requireEnd("Trailing content after the top-level object"); + + if (modelType != null && !"Unigram".equals(modelType)) { + throw new IllegalArgumentException(file + " has a '" + modelType + "' tokenizer model; " + + "only the Unigram list layout maps pieces to matrix rows here. For a WordPiece " + + "model, load from its vocab.txt instead"); + } + if (vocab == null) { + throw new IllegalArgumentException(file + " has no model.vocab list; it does not name " + + "the matrix rows"); + } + return overlayAddedTokens(vocab, addedTokens, file); + } + + /** The fields read out of the {@code model} object. */ + private record ParsedModel(String type, List vocab) { + } + + /** + * Parses the {@code model} object, collecting its {@code type} and its {@code vocab} pieces + * in list order. + * + * @param cursor The cursor, positioned at the object's opening brace. + * @return The parsed type and vocabulary; either may be absent ({@code null}). + */ + private static ParsedModel parseModel(JsonCursor cursor) { + cursor.expect('{'); + cursor.skipWhitespace(); + String type = null; + List vocab = null; + if (cursor.peek() == '}') { + cursor.consume(); + return new ParsedModel(null, null); + } + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "type" -> { + if (type != null) { + throw cursor.malformed("Field 'model.type' appears more than once"); + } + type = cursor.parseString(); + } + case "vocab" -> { + if (vocab != null) { + throw cursor.malformed("Field 'model.vocab' appears more than once"); + } + if (cursor.peek() == '{') { + throw cursor.malformed("model.vocab is an object; only the Unigram list layout " + + "([piece, score] pairs) maps pieces to matrix rows here"); + } + vocab = parseVocabList(cursor); + } + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return new ParsedModel(type, vocab); + } + throw cursor.malformed("Expected ',' or '}' after a model field, got '" + next + "'"); + } + } + + /** + * Parses the Unigram {@code vocab} list of {@code [piece, score]} pairs. + * + * @param cursor The cursor, positioned at the list's opening bracket. + * @return The pieces in list order. + */ + private static List parseVocabList(JsonCursor cursor) { + cursor.expect('['); + cursor.skipWhitespace(); + final List pieces = new ArrayList<>(); + if (cursor.peek() == ']') { + cursor.consume(); + return pieces; + } + while (true) { + cursor.skipWhitespace(); + cursor.expect('['); + cursor.skipWhitespace(); + pieces.add(cursor.parseString()); + cursor.skipWhitespace(); + cursor.expect(','); + cursor.skipWhitespace(); + cursor.skipValue(); + cursor.skipWhitespace(); + cursor.expect(']'); + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == ']') { + return pieces; + } + throw cursor.malformed("Expected ',' or ']' after a vocab entry, got '" + next + "'"); + } + } + + /** + * Parses the {@code added_tokens} list of objects, keeping each entry's {@code id} and + * {@code content}. + * + * @param cursor The cursor, positioned at the list's opening bracket. + * @return The added tokens in list order. + */ + private static List parseAddedTokens(JsonCursor cursor) { + cursor.expect('['); + cursor.skipWhitespace(); + final List tokens = new ArrayList<>(); + if (cursor.peek() == ']') { + cursor.consume(); + return tokens; + } + while (true) { + cursor.skipWhitespace(); + tokens.add(parseAddedToken(cursor)); + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == ']') { + return tokens; + } + throw cursor.malformed("Expected ',' or ']' after an added token, got '" + next + "'"); + } + } + + /** + * Parses one {@code added_tokens} object, requiring its {@code id} and {@code content}. + * + * @param cursor The cursor, positioned at the object's opening brace. + * @return The parsed entry. + */ + private static AddedToken parseAddedToken(JsonCursor cursor) { + cursor.expect('{'); + cursor.skipWhitespace(); + Long id = null; + String content = null; + if (cursor.peek() == '}') { + throw cursor.malformed("An added token must carry 'id' and 'content'"); + } + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "id" -> { + if (id != null) { + throw cursor.malformed("Field 'id' appears more than once in an added token"); + } + id = cursor.parseLong(); + } + case "content" -> { + if (content != null) { + throw cursor.malformed("Field 'content' appears more than once in an added token"); + } + content = cursor.parseString(); + } + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after an added token field, got '" + + next + "'"); + } + if (id == null || content == null) { + throw cursor.malformed("An added token must carry 'id' and 'content'"); + } + if (id < 0) { + throw cursor.malformed("An added token's id must not be negative: " + id); + } + return new AddedToken(id, content); + } + + /** + * Overlays the added tokens onto the vocabulary in id order: an id equal to the current size + * appends, an id below it must agree with the piece already there, and a gap fails loud. + * + * @param vocab The {@code model.vocab} pieces in list order; extended in place. + * @param addedTokens The added tokens to overlay. + * @param file The source file, for error messages. + * @return The vocabulary with the added tokens applied. + */ + private static List overlayAddedTokens(List vocab, + List addedTokens, Path file) { + final List byId = new ArrayList<>(addedTokens); + byId.sort(Comparator.comparingLong(AddedToken::id)); + for (final AddedToken token : byId) { + if (token.id() == vocab.size()) { + vocab.add(token.content()); + } else if (token.id() < vocab.size()) { + final String existing = vocab.get((int) token.id()); + if (!existing.equals(token.content())) { + throw new IllegalArgumentException(file + " declares added token '" + token.content() + + "' at id " + token.id() + " but model.vocab holds '" + existing + + "' there; the file contradicts itself"); + } + } else { + throw new IllegalArgumentException(file + " declares added token '" + token.content() + + "' at id " + token.id() + " but the vocabulary only has " + vocab.size() + + " rows; the id space has a gap"); + } + } + return vocab; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java deleted file mode 100644 index 8adffbac51..0000000000 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/WordpiecePipeline.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package opennlp.embeddings; - -import java.text.Normalizer; -import java.util.Locale; -import java.util.Objects; -import java.util.Set; - -import opennlp.tools.tokenize.WordpieceTokenizer; - -/** - * The full BERT tokenization pipeline used for embedding-table lookup: basic tokenization - * (control removal, whitespace normalization, CJK isolation, optional lower casing with accent - * stripping, punctuation isolation) followed by {@link WordpieceTokenizer} with the BERT special - * tokens and the 100-character word limit. It produces pieces without offset bookkeeping. - */ -final class WordpiecePipeline { - - private static final int MAX_WORD_CHARACTERS = 100; - - private final WordpieceTokenizer wordpieceTokenizer; - private final boolean lowerCase; - - /** - * @param vocabulary The wordpiece vocabulary. Must not be {@code null}. - * @param lowerCase Whether basic tokenization lower-cases and strips accents. - */ - WordpiecePipeline(Set vocabulary, boolean lowerCase) { - Objects.requireNonNull(vocabulary, "vocabulary must not be null"); - this.wordpieceTokenizer = new WordpieceTokenizer(vocabulary, - WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, - WordpieceTokenizer.BERT_UNK_TOKEN, MAX_WORD_CHARACTERS); - this.lowerCase = lowerCase; - } - - /** - * {@return the wordpiece tokens of {@code text}, wrapped in {@code [CLS]} and {@code [SEP]}} - * - * @param text The text to tokenize. - */ - String[] tokenize(String text) { - return wordpieceTokenizer.tokenize(normalize(text)); - } - - /** {@return {@code text} after BERT basic tokenization} */ - private String normalize(String text) { - String normalized = cleanText(text); - normalized = isolateCjkCharacters(normalized); - if (lowerCase) { - normalized = stripAccents(normalized.toLowerCase(Locale.ROOT)); - } - return isolatePunctuation(normalized); - } - - /** {@return {@code text} with null, replacement, and control characters removed} */ - private static String cleanText(String text) { - final StringBuilder cleaned = new StringBuilder(text.length()); - text.codePoints().forEach(codePoint -> { - if (codePoint == 0 || codePoint == 0xFFFD || isControl(codePoint)) { - return; - } - if (isWhitespace(codePoint)) { - cleaned.append(' '); - } else { - cleaned.appendCodePoint(codePoint); - } - }); - return cleaned.toString(); - } - - /** {@return {@code text} with each CJK character surrounded by spaces} */ - private static String isolateCjkCharacters(String text) { - final StringBuilder spaced = new StringBuilder(text.length()); - text.codePoints().forEach(codePoint -> { - if (isCjk(codePoint)) { - spaced.append(' ').appendCodePoint(codePoint).append(' '); - } else { - spaced.appendCodePoint(codePoint); - } - }); - return spaced.toString(); - } - - /** {@return {@code text} with combining accent marks removed} */ - private static String stripAccents(String text) { - final String decomposed = Normalizer.normalize(text, Normalizer.Form.NFD); - final StringBuilder stripped = new StringBuilder(decomposed.length()); - decomposed.codePoints().forEach(codePoint -> { - if (Character.getType(codePoint) != Character.NON_SPACING_MARK) { - stripped.appendCodePoint(codePoint); - } - }); - return stripped.toString(); - } - - /** {@return {@code text} with each punctuation character surrounded by spaces} */ - private static String isolatePunctuation(String text) { - final StringBuilder spaced = new StringBuilder(text.length()); - text.codePoints().forEach(codePoint -> { - if (isPunctuation(codePoint)) { - spaced.append(' ').appendCodePoint(codePoint).append(' '); - } else { - spaced.appendCodePoint(codePoint); - } - }); - return spaced.toString(); - } - - /** {@return whether the code point is a control character, treating tab/newline/return as not} */ - private static boolean isControl(int codePoint) { - if (codePoint == '\t' || codePoint == '\n' || codePoint == '\r') { - return false; - } - return switch (Character.getType(codePoint)) { - case Character.CONTROL, Character.FORMAT, Character.SURROGATE, - Character.PRIVATE_USE, Character.UNASSIGNED -> true; - default -> false; - }; - } - - /** {@return whether the code point is whitespace for BERT basic tokenization} */ - private static boolean isWhitespace(int codePoint) { - if (codePoint == ' ' || codePoint == '\t' || codePoint == '\n' || codePoint == '\r') { - return true; - } - return Character.getType(codePoint) == Character.SPACE_SEPARATOR; - } - - /** {@return whether the code point is punctuation for BERT basic tokenization} */ - private static boolean isPunctuation(int codePoint) { - if ((codePoint >= 33 && codePoint <= 47) || (codePoint >= 58 && codePoint <= 64) - || (codePoint >= 91 && codePoint <= 96) || (codePoint >= 123 && codePoint <= 126)) { - return true; - } - return switch (Character.getType(codePoint)) { - case Character.CONNECTOR_PUNCTUATION, Character.DASH_PUNCTUATION, - Character.START_PUNCTUATION, Character.END_PUNCTUATION, - Character.INITIAL_QUOTE_PUNCTUATION, Character.FINAL_QUOTE_PUNCTUATION, - Character.OTHER_PUNCTUATION -> true; - default -> false; - }; - } - - /** {@return whether the code point is a CJK ideograph} */ - private static boolean isCjk(int codePoint) { - return (codePoint >= 0x4E00 && codePoint <= 0x9FFF) - || (codePoint >= 0x3400 && codePoint <= 0x4DBF) - || (codePoint >= 0x20000 && codePoint <= 0x2A6DF) - || (codePoint >= 0x2A700 && codePoint <= 0x2B73F) - || (codePoint >= 0x2B740 && codePoint <= 0x2B81F) - || (codePoint >= 0x2B820 && codePoint <= 0x2CEAF) - || (codePoint >= 0xF900 && codePoint <= 0xFAFF) - || (codePoint >= 0x2F800 && codePoint <= 0x2FA1F); - } -} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpieceVocabularyTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingVocabularyTest.java similarity index 84% rename from opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpieceVocabularyTest.java rename to opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingVocabularyTest.java index 0609efa834..92ae7ee68f 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpieceVocabularyTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingVocabularyTest.java @@ -32,12 +32,12 @@ * The vocabulary contract: line number is the id, duplicates fail loud, the id lookup uses a * {@code -1} sentinel, and the reverse lookup enforces its bounds. */ -class WordpieceVocabularyTest { +class EmbeddingVocabularyTest { @Test void testLineNumberIsTheTokenId() { - final WordpieceVocabulary vocabulary = - WordpieceVocabulary.fromLines(List.of("[CLS]", "[SEP]", "hello", "world"), "test"); + final EmbeddingVocabulary vocabulary = + EmbeddingVocabulary.fromLines(List.of("[CLS]", "[SEP]", "hello", "world"), "test"); assertEquals(4, vocabulary.size()); assertEquals(0, vocabulary.id("[CLS]")); assertEquals(2, vocabulary.id("hello")); @@ -47,8 +47,8 @@ void testLineNumberIsTheTokenId() { @Test void testUnknownTokenIdIsTheSentinel() { - final WordpieceVocabulary vocabulary = - WordpieceVocabulary.fromLines(List.of("hello"), "test"); + final EmbeddingVocabulary vocabulary = + EmbeddingVocabulary.fromLines(List.of("hello"), "test"); assertEquals(-1, vocabulary.id("missing")); assertThrows(IllegalArgumentException.class, () -> vocabulary.id(null)); } @@ -56,15 +56,15 @@ void testUnknownTokenIdIsTheSentinel() { @Test void testDuplicateTokenFailsLoudlyNamingBothLines() { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> WordpieceVocabulary.fromLines(List.of("hello", "world", "hello"), "test")); + () -> EmbeddingVocabulary.fromLines(List.of("hello", "world", "hello"), "test")); assertTrue(e.getMessage().contains("hello"), e.getMessage()); assertTrue(e.getMessage().contains("0") && e.getMessage().contains("2"), e.getMessage()); } @Test void testReverseLookupEnforcesBounds() { - final WordpieceVocabulary vocabulary = - WordpieceVocabulary.fromLines(List.of("hello"), "test"); + final EmbeddingVocabulary vocabulary = + EmbeddingVocabulary.fromLines(List.of("hello"), "test"); assertEquals("hello", vocabulary.token(0)); assertThrows(IllegalArgumentException.class, () -> vocabulary.token(-1)); assertThrows(IllegalArgumentException.class, () -> vocabulary.token(1)); @@ -74,7 +74,7 @@ void testReverseLookupEnforcesBounds() { void testReadFromFileMatchesInMemoryLines(@TempDir Path dir) throws IOException { final Path file = dir.resolve("vocab.txt"); Files.write(file, List.of("[CLS]", "token")); - final WordpieceVocabulary read = WordpieceVocabulary.read(file); + final EmbeddingVocabulary read = EmbeddingVocabulary.fromVocabTxt(file); assertEquals(2, read.size()); assertEquals(1, read.id("token")); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java new file mode 100644 index 0000000000..c1e53cec90 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.embeddings.StaticEmbeddingModel.Normalization; +import opennlp.subword.sentencepiece.SentencePieceTokenizer; +import opennlp.tools.tokenize.SubwordPiece; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The SentencePiece loading path, exercised against a real trained tiny model (a copy of the + * opennlp-subword test fixture). The matrix vocabulary is written the way a distillation ships + * it: control pieces dropped, rows ordered differently from the tokenizer's ids, extra special + * rows in front, and an extra token appended through {@code added_tokens}; every lookup must + * therefore go by piece string, never by tokenizer id. + */ +class StaticEmbeddingModelSentencePieceTest { + + private static final String MODEL_RESOURCE = "/opennlp/embeddings/tiny-unigram.model"; + private static final int DIMENSION = 4; + + private static byte[] modelBytes; + private static SentencePieceTokenizer tokenizer; + // The matrix rows: , , then every poolable tokenizer piece, then . + private static List rows; + + @BeforeAll + static void loadFixture() throws IOException { + try (InputStream in = + StaticEmbeddingModelSentencePieceTest.class.getResourceAsStream(MODEL_RESOURCE)) { + modelBytes = in.readAllBytes(); + } + tokenizer = SentencePieceTokenizer.load( + StaticEmbeddingModelSentencePieceTest.class.getResourceAsStream(MODEL_RESOURCE)); + rows = new ArrayList<>(); + rows.add(""); + rows.add(""); + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + if (!tokenizer.isControl(id) && !tokenizer.isUnknown(id)) { + rows.add(tokenizer.idToPiece(id)); + } + } + } + + /** + * {@return the value at {@code (row, d)} of the deterministic test matrix} + * + * @param row The matrix row. + * @param d The dimension index. + */ + private static float cell(int row, int d) { + return row + d * 0.25f; + } + + /** + * Writes the three SentencePiece-layout files (and optionally a {@code config.json}) into a + * directory: the copied {@code .model}, a synthesized Unigram {@code tokenizer.json} whose + * vocabulary is {@link #rows} with one token appended via {@code added_tokens}, and a + * deterministic embedding matrix with one extra row for it. + * + * @param dir The directory to write into. + * @param normalize The {@code config.json} normalize value, or {@code null} to omit the file. + * @return The directory. + * @throws IOException Thrown if writing fails. + */ + private static Path writeModelDirectory(Path dir, Boolean normalize) throws IOException { + Files.write(dir.resolve("sentencepiece.bpe.model"), modelBytes); + Files.writeString(dir.resolve("tokenizer.json"), tokenizerJson(rows)); + final float[][] matrix = new float[rows.size() + 1][DIMENSION]; + for (int row = 0; row < matrix.length; row++) { + for (int d = 0; d < DIMENSION; d++) { + matrix[row][d] = cell(row, d); + } + } + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", matrix)); + if (normalize != null) { + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":" + normalize + "}"); + } + return dir; + } + + /** + * {@return a Unigram {@code tokenizer.json} whose vocabulary is the given pieces plus an + * appended added token} + * + *

The appended token is not named {@code } because the fixture model itself defines + * {@code } as a user-defined piece, which already owns a row.

+ * + * @param pieces The {@code model.vocab} pieces in row order. + */ + private static String tokenizerJson(List pieces) { + final StringBuilder json = new StringBuilder("{\"version\":\"1.0\",\"added_tokens\":["); + json.append("{\"id\":0,\"content\":\"\",\"special\":true},"); + json.append("{\"id\":").append(pieces.size()).append(",\"content\":\"\"," + + "\"special\":true}],"); + json.append("\"normalizer\":{\"type\":\"Precompiled\"},\"model\":{\"type\":\"Unigram\"," + + "\"unk_id\":1,\"vocab\":["); + for (int i = 0; i < pieces.size(); i++) { + if (i > 0) { + json.append(','); + } + json.append('[').append(quote(pieces.get(i))).append(",-").append(i % 7).append(".5]"); + } + return json.append("]}}").toString(); + } + + /** {@return {@code s} as a JSON string literal} */ + private static String quote(String s) { + final StringBuilder quoted = new StringBuilder("\""); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + switch (c) { + case '"' -> quoted.append("\\\""); + case '\\' -> quoted.append("\\\\"); + default -> { + if (c < 0x20) { + quoted.append(String.format("\\u%04x", (int) c)); + } else { + quoted.append(c); + } + } + } + } + return quoted.append('"').toString(); + } + + @Test + void testEmbedGathersRowsByPieceStringAcrossTheIdOffset(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = loadFromDirectory(writeModelDirectory(dir, null)); + + // "a" segments to the single piece "▁a"; the embedding must be exactly that piece's matrix + // row, found by string in the reordered vocabulary, not by the tokenizer's id. + final List pieces = tokenizer.encode("a"); + assertEquals(1, pieces.size()); + final int row = rows.indexOf(pieces.get(0).piece()); + assertTrue(row >= 2, "the fixture row must sit above the injected specials"); + final float[] expected = new float[DIMENSION]; + for (int d = 0; d < DIMENSION; d++) { + expected[d] = cell(row, d); + } + assertArrayEquals(expected, model.embed("a"), 1e-5f); + } + + @Test + void testEmbedMeanPoolsAllMappedPieces(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = loadFromDirectory(writeModelDirectory(dir, null)); + + // Expected: the mean over every non-control, non-unknown piece's row, resolved by string. + final List pieces = tokenizer.encode("Hello world"); + final float[] expected = new float[DIMENSION]; + int pooled = 0; + for (final SubwordPiece piece : pieces) { + if (tokenizer.isControl(piece.id()) || tokenizer.isUnknown(piece.id())) { + continue; + } + final int row = rows.indexOf(piece.piece()); + assertTrue(row >= 0, "fixture piece '" + piece.piece() + "' must have a row"); + for (int d = 0; d < DIMENSION; d++) { + expected[d] += cell(row, d); + } + pooled++; + } + assertTrue(pooled > 1, "the fixture text must pool more than one piece"); + for (int d = 0; d < DIMENSION; d++) { + expected[d] /= pooled; + } + assertArrayEquals(expected, model.embed("Hello world"), 1e-4f); + } + + @Test + void testUnknownPiecesAreSkippedInPooling(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = loadFromDirectory(writeModelDirectory(dir, null)); + + // The euro sign is outside the tiny training corpus, so it segments to the dummy-prefix + // piece plus an unknown piece carrying the surface text. The unknown piece's string is not + // a vocabulary entry, so pooling must skip it by its id, leaving only the mapped pieces. + final List pieces = tokenizer.encode("€"); + final float[] expected = new float[DIMENSION]; + int pooled = 0; + int unknown = 0; + for (final SubwordPiece piece : pieces) { + if (tokenizer.isUnknown(piece.id())) { + unknown++; + continue; + } + if (tokenizer.isControl(piece.id())) { + continue; + } + final int row = rows.indexOf(piece.piece()); + for (int d = 0; d < DIMENSION; d++) { + expected[d] += cell(row, d); + } + pooled++; + } + assertTrue(unknown > 0, "fixture assumption: '€' must produce an unknown piece"); + for (int d = 0; d < DIMENSION; d++) { + expected[d] /= Math.max(pooled, 1); + } + assertArrayEquals(expected, model.embed("€"), 1e-5f); + } + + @Test + void testDirectoryLoadDetectsTheSentencePieceLayout(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeModelDirectory(dir, true)); + + assertEquals(DIMENSION, model.dimension()); + assertEquals(rows.size() + 1, model.vocabularySize()); + // normalize=true from config.json: the pooled vector must have unit length. + final float[] vector = model.embed("a"); + double normSquared = 0; + for (final float v : vector) { + normSquared += (double) v * v; + } + assertEquals(1.0, Math.sqrt(normSquared), 1e-5); + } + + @Test + void testMostSimilarNeverReturnsSpecialRows(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = loadFromDirectory(writeModelDirectory(dir, null)); + + for (final Neighbor neighbor : model.mostSimilar("a", 5)) { + assertFalse(List.of("", "", "").contains(neighbor.token()), + "special row leaked into neighbors: " + neighbor.token()); + } + } + + @Test + void testLoadRejectsAVocabularyMissingAPoolablePiece(@TempDir Path dir) throws IOException { + writeModelDirectory(dir, null); + // Remove one poolable piece from the matrix vocabulary; the matrix shrinks with it, so only + // the coverage check can catch the mismatch. + final List truncated = new ArrayList<>(rows); + truncated.remove(truncated.size() - 1); + Files.writeString(dir.resolve("tokenizer.json"), tokenizerJson(truncated)); + final float[][] matrix = new float[truncated.size() + 1][DIMENSION]; + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", matrix)); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> loadFromDirectory(dir)); + assertTrue(e.getMessage().contains("do not belong"), e.getMessage()); + } + + @Test + void testLoadRejectsARowCountMismatch(@TempDir Path dir) throws IOException { + writeModelDirectory(dir, null); + final float[][] matrix = new float[rows.size()][DIMENSION]; + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", matrix)); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> loadFromDirectory(dir)); + assertTrue(e.getMessage().contains("rows"), e.getMessage()); + } + + @Test + void testDirectoryLoadNamesTheMissingSentencePieceModel(@TempDir Path dir) throws IOException { + writeModelDirectory(dir, true); + Files.delete(dir.resolve("sentencepiece.bpe.model")); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + assertTrue(e.getMessage().contains("copy the .model"), e.getMessage()); + } + + @Test + void testLoadSentencePieceRejectsNullArguments(@TempDir Path dir) throws IOException { + writeModelDirectory(dir, null); + final Path model = dir.resolve("sentencepiece.bpe.model"); + final Path json = dir.resolve("tokenizer.json"); + final Path tensors = dir.resolve("model.safetensors"); + + assertThrows(IllegalArgumentException.class, () -> + StaticEmbeddingModel.loadSentencePiece(null, json, tensors, Normalization.NONE)); + assertThrows(IllegalArgumentException.class, () -> + StaticEmbeddingModel.loadSentencePiece(model, null, tensors, Normalization.NONE)); + assertThrows(IllegalArgumentException.class, () -> + StaticEmbeddingModel.loadSentencePiece(model, json, null, Normalization.NONE)); + assertThrows(IllegalArgumentException.class, () -> + StaticEmbeddingModel.loadSentencePiece(model, json, tensors, null)); + } + + /** + * Loads through the explicit SentencePiece overload from a directory written by + * {@link #writeModelDirectory(Path, Boolean)}. + * + * @param dir The model directory. + * @return The loaded model. + * @throws IOException Thrown if reading fails. + */ + private static StaticEmbeddingModel loadFromDirectory(Path dir) throws IOException { + return StaticEmbeddingModel.loadSentencePiece(dir.resolve("sentencepiece.bpe.model"), + dir.resolve("tokenizer.json"), dir.resolve("model.safetensors"), Normalization.NONE); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index a579a047f0..da17b2c656 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -300,7 +300,7 @@ void testDirectoryLoadNamesTheMissingFile(@TempDir Path dir) throws IOException final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); assertTrue(e.getMessage().contains("config.json")); - assertTrue(e.getMessage().contains("load(vocabularyFile, safetensorsFile")); + assertTrue(e.getMessage().contains("explicit load overloads")); } @Test diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java new file mode 100644 index 0000000000..043f97a376 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The {@code tokenizer.json} vocabulary contract: the Unigram {@code model.vocab} list order is + * the row order, {@code added_tokens} append or must agree, everything else is skipped, and any + * departure from the known shape fails loud. + */ +class TokenizerJsonVocabTest { + + @TempDir + private Path dir; + + private Path write(String json) throws IOException { + final Path file = dir.resolve("tokenizer.json"); + Files.writeString(file, json); + return file; + } + + @Test + void testVocabListOrderIsTheRowOrder() throws IOException { + final Path file = write("{\"model\":{\"type\":\"Unigram\",\"unk_id\":1," + + "\"vocab\":[[\"\",0.0],[\"\",0.0],[\"\\u2581a\",-2.5],[\"b\",-3.0]]}}"); + + assertEquals(List.of("", "", "▁a", "b"), TokenizerJsonVocab.rows(file)); + } + + @Test + void testAddedTokenAtTheNextRowAppends() throws IOException { + final Path file = write("{\"added_tokens\":[{\"id\":2,\"content\":\"\"," + + "\"special\":true}]," + + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0],[\"b\",-1.0]]}}"); + + assertEquals(List.of("a", "b", ""), TokenizerJsonVocab.rows(file)); + } + + @Test + void testAddedTokenAtAnExistingRowMustAgree() throws IOException { + final Path agreeing = write("{\"added_tokens\":[{\"id\":0,\"content\":\"\"}]," + + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"\",0.0],[\"a\",-1.0]]}}"); + assertEquals(List.of("", "a"), TokenizerJsonVocab.rows(agreeing)); + + final Path contradicting = write("{\"added_tokens\":[{\"id\":0,\"content\":\"\"}]," + + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"\",0.0],[\"a\",-1.0]]}}"); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(contradicting)); + assertTrue(e.getMessage().contains("contradicts"), e.getMessage()); + } + + @Test + void testAddedTokenBeyondTheNextRowIsAGap() throws IOException { + final Path file = write("{\"added_tokens\":[{\"id\":5,\"content\":\"\"}]," + + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]]}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(file)); + assertTrue(e.getMessage().contains("gap"), e.getMessage()); + } + + @Test + void testAddedTokensAreOverlaidInIdOrderNotListOrder() throws IOException { + final Path file = write("{\"added_tokens\":[{\"id\":3,\"content\":\"y\"}," + + "{\"id\":2,\"content\":\"x\"}]," + + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0],[\"b\",-1.0]]}}"); + + assertEquals(List.of("a", "b", "x", "y"), TokenizerJsonVocab.rows(file)); + } + + @Test + void testSkipsUnrelatedSectionsAndDecodesEscapes() throws IOException { + final Path file = write("{\"version\":\"1.0\",\"truncation\":null," + + "\"normalizer\":{\"type\":\"Precompiled\",\"precompiled_charsmap\":\"AAAA\"}," + + "\"pre_tokenizer\":[1,2,{\"a\":[true,false]}]," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":0," + + "\"vocab\":[[\"\\\"quoted\\\"\",0.0],[\"tab\\there\",-1.0]]}}"); + + assertEquals(List.of("\"quoted\"", "tab\there"), TokenizerJsonVocab.rows(file)); + } + + @Test + void testRejectsANonUnigramModel() throws IOException { + final Path file = write("{\"model\":{\"type\":\"BPE\",\"vocab\":[[\"a\",0.0]]}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(file)); + assertTrue(e.getMessage().contains("BPE"), e.getMessage()); + } + + @Test + void testRejectsAnObjectShapedVocab() throws IOException { + // The WordPiece/BPE tokenizer.json layout stores vocab as {piece: id}; ids in that shape + // are not list positions, so it must be refused rather than misread. + final Path file = write("{\"model\":{\"type\":\"Unigram\",\"vocab\":{\"a\":0,\"b\":1}}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(file)); + assertTrue(e.getMessage().contains("object"), e.getMessage()); + } + + @Test + void testRejectsAMissingVocab() throws IOException { + final Path noModel = write("{\"version\":\"1.0\"}"); + assertTrue(assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(noModel)).getMessage().contains("model.vocab")); + + final Path noVocab = write("{\"model\":{\"type\":\"Unigram\"}}"); + assertTrue(assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(noVocab)).getMessage().contains("model.vocab")); + } + + @Test + void testRejectsAnAddedTokenWithoutIdOrContent() throws IOException { + final Path file = write("{\"added_tokens\":[{\"content\":\"\"}]," + + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]]}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(file)); + assertTrue(e.getMessage().contains("id"), e.getMessage()); + } + + @Test + void testRejectsDuplicateTopLevelSections() throws IOException { + final Path file = write("{\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]]}," + + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"b\",0.0]]}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(file)); + assertTrue(e.getMessage().contains("more than once"), e.getMessage()); + } + + @Test + void testRejectsMalformedJson() throws IOException { + final Path file = write("{\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]"); + + assertThrows(IllegalArgumentException.class, () -> TokenizerJsonVocab.rows(file)); + } + + @Test + void testVocabularyEntryPointRejectsDuplicatePieces() throws IOException { + final Path file = write("{\"model\":{\"type\":\"Unigram\"," + + "\"vocab\":[[\"a\",0.0],[\"a\",-1.0]]}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> EmbeddingVocabulary.fromTokenizerJson(file)); + assertTrue(e.getMessage().contains("more than once"), e.getMessage()); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java deleted file mode 100644 index 2569d5f0a1..0000000000 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/WordpiecePipelineTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package opennlp.embeddings; - -import java.util.Set; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * Pins the module-internal tokenization pipeline against reference token sequences. - *

- * All expected sequences were generated with the HuggingFace {@code tokenizers} reference - * implementation ({@code BertWordPieceTokenizer}) using the same vocabulary, so the lookup - * path is verified to be identical to the reference BERT tokenization. - */ -class WordpiecePipelineTest { - - private static final Set VOCABULARY = Set.of( - "the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog", - "em", "##bed", "##ding", "##s", - "wurttemberg", "strasse", "grosse", - "don", "t", "wait", "what", ".", ",", "?", "!", "'", - "\u6211", "\u7231", // CJK - "natural", "language", "processing"); - - @Test - void testLowerCasesCapitalizedWords() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - final String[] tokens = - pipeline.tokenize("The quick brown fox jumps over the lazy dog."); - - final String[] expected = {"[CLS]", "the", "quick", "brown", "fox", "jumps", "over", - "the", "lazy", "dog", ".", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testLowerCasesBeforeWordpieceSplitting() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - final String[] tokens = pipeline.tokenize("Embeddings"); - - final String[] expected = {"[CLS]", "em", "##bed", "##ding", "##s", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testStripsAccentsButKeepsNonCombiningCharacters() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - // The u-umlaut decomposes to u plus a combining diaeresis and the mark is stripped; - // the sharp s is not a combining mark and must survive, leaving an OOV token. - final String[] tokens = pipeline.tokenize("W\u00fcrttemberg Stra\u00dfe"); - - final String[] expected = {"[CLS]", "wurttemberg", "[UNK]", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testSplitsPunctuationRunsIntoSingleCharacters() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - final String[] tokens = pipeline.tokenize("Wait... what?!"); - - final String[] expected = {"[CLS]", "wait", ".", ".", ".", "what", "?", "!", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testSplitsApostrophesAsPunctuation() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - final String[] tokens = pipeline.tokenize("don't"); - - final String[] expected = {"[CLS]", "don", "'", "t", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testIsolatesCjkIdeographs() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - final String[] tokens = pipeline.tokenize("\u6211\u7231natural language processing"); - - final String[] expected = {"[CLS]", "\u6211", "\u7231", "natural", "language", - "processing", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testCleansControlCharactersAndNormalizesWhitespace() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - // Tab and no-break space are whitespace; the NUL character is removed, - // joining "brown" and "fox" into one out-of-vocabulary token. - final String[] tokens = pipeline.tokenize("the\tquick\u00a0brown\u0000fox"); - - final String[] expected = {"[CLS]", "the", "quick", "[UNK]", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testRemovesPrivateUseAndUnassignedCharacters() { - final WordpiecePipeline pipeline = new WordpiecePipeline(VOCABULARY, true); - // The reference implementation treats all C* categories as control - // characters: private use (U+E000, Co) and noncharacters (U+FDD0, Cn) - // are removed, joining the surrounding text into one OOV token. - final String[] tokens = pipeline.tokenize("fox\ue000jumps and fox\ufdd0jumps"); - - final String[] expected = {"[CLS]", "[UNK]", "[UNK]", "[UNK]", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testCasedModeKeepsCaseAndAccents() { - final WordpiecePipeline pipeline = new WordpiecePipeline( - Set.of("The", "W\u00fcrttemberg", "fox"), false); - final String[] tokens = pipeline.tokenize("The W\u00fcrttemberg fox"); - - final String[] expected = - {"[CLS]", "The", "W\u00fcrttemberg", "fox", "[SEP]"}; - Assertions.assertArrayEquals(expected, tokens); - } - - @Test - void testRejectsNullVocabulary() { - Assertions.assertThrows(NullPointerException.class, () -> new WordpiecePipeline(null, true)); - } -} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/resources/opennlp/embeddings/tiny-unigram.model b/opennlp-extensions/opennlp-embeddings/src/test/resources/opennlp/embeddings/tiny-unigram.model new file mode 100644 index 0000000000000000000000000000000000000000..b6e30611e409fcb4ae76b41b8abe0630ecac2f14 GIT binary patch literal 245202 zcmZU*3s{ubxv>4r3vbWRks{sjWupVjWujzjoVNpN)Zu(7;BIj z&O|LCNQ^Pon8|P&h5?7+G%youj9Q1PHPrYSwz7#;xB3xl+{PNW|Gj1ud;K5Rbzjf3 z*1OJYJ?r$o!=TVXfsvW(lA|@@?{WR0A%hH&>GxfRK_Pl_5ISHG zv6sjF>ZKL_*a3qE4Kn=K`zFoa6!6)g0fThu+QC<*`?Q~EN4}5sX^C1)-OE00oi?F! z<7a~g4+Ksg*wQj!#R{>_fMH zHfZ1=ec}eq;Y20dRBipx?S9Wy+QFT__jzv827R&PGgZQ*b*r^6ZtU^}h+nHMc=>*J@)v^=r>)JEJT}dXyrM{v4<3~1M==cn6aHicCoOJ{uDN7_5X1BN|>x=yxsmF zNa8x}f8OnPQNGoG?{}qTYF|Cp@t`gn<1-kJpO3=BU;OoJ<2PwvJcV18He-`^G#1@o z#EnVXnWeZW``T67AC3M%@$1)XTV8fQX#NDP@Vn3YW5jD2yAJvaxKZ2t^UojnGfo}} zI`r9~#|DMbpy`RLQxex@#QU1^%aQmG9W+q+Z%X-1>RQd`^Pq84SNYv^%1c_83hVEb z^+|~diP~S4hd<~W@ig0Q<)*mUQRox>k2c1yP1gQZx&8H88?POoi;Eh^CnRgSHT_Q* z=nI;0`As}?U6S_QA90dm<_4|zFa6=x$7>gU_k}No)tJLemFWJolQOg|P5vkGSlvKZ z|C2XTwZzYH`Dc}vwdOYuKZul&su})%E+N_Vg4{HjS)jVCO{*t`0L3?KDasPuvtsn!HMtn>Adxc+{s{QVBzxJH= zw$iBOiI#r#q|vVuXRMa3p40Ga;y1)6s1fAz;L&gWnm@<5b$*w>Hxp7ftj|pUXP=b# z4apDUJ~}9b9$%NbAtioo((fMl(j&db9Df{ti1iy%e-rT(s}%4e=?OQ#tMr^7y0;+JQs3?^|@@hSiCh?F)b0^!4$i^xhF% z`7%s}Qj;g#Hj_yzLjScZy&dWGS zQcX3QqeC2D`0_Lm2xOV89ol5r6|bwh?0kmc9YnceiC-uEZQ?;kzo%Z)PT*-;6; zp1EmvY}bQ?QWCX&yT9^1iO)>XMxQ+68hi?8KU(|R!4}N{WHB;7WcmJdNLEKfT zTGIhs{)V9P+CSd-PhX^@1TE?dRPsy0P{MEdz2A64yZpa?Z#90Nb)NI3oA@TfZO~^p z7#&H!^G`$9m79h-_$g+A>K{|{QZNEsjP#!}D*alFUsHh_`qdNrAF9rWYWqXg?N|NA zk+Eh&V*Dz<&+tJ*=!cBdwkN{uwTEL z*W%axC10`zdm$bFfX_%up{slz_bawm%X|3vjrf$LwOZjrkMzX&^flUU<>4deZ83+ULsgU|wfhJ9*UqFeRSZHuxBh`}NwOJU;dId!%KiGKyz)_DA0kzb;vO z^*C<-+H=2|{hj`!6a5j@pxC7S>?9sE@){QWqe}O;*2Xp3U%GJJZ;SQo*V3AQJL?ad z62DIS?bm)iiCxEQm;8GCx_B-7TXcVg)gI@Q?{HFKwfbrA8O}dQ|BWOqcOWi*jW=Yj zTc=e$=6{-)mWc(`D3|}~+EgZ-+>!lmmgBYBpGNybGmh3IX@CAPE^3iLDI=fh51FAh z7GM7qH$_;lY&UwQKVV`)hIV5%y1%+9saj^FKUn;RqDa9|H_YG3I8 z?$ds$Jr#Jzr%hvpyXVWg*JORD=Xf6Kx%Pk3+xzuz+G_jtzwK%0*DvmJ_3Kyi%e4*n z)4y6?)URJF|G=+Ps;995N`N-8Uz^^q&Fa@?_iHcqYrpE(mijei;mW{BP^lk3RI47U z)eluyzj|WtLsiWY{uEA{9;#}Q^EcIh?fJA{?=?O2?=|iEC;$5i`X+$7hyFPa!++EA ze!u@WEw+bx$Nhp6_kyLP-~U2=X}|urotB6C2lj{hN45R>#qw5 zVVVEik=O5kWoywt=@09B^}`+g{#XBy->>(zl=yW<>bH9f@3*nvQS(nvwwM2t;{)SA zIX=AaU?e=N72F>ai?wYJI&*KodZN*^)q)2DG$R7rZaUXmNx#2_}Nj#V_MvPNnKE!}=D*jMOxMF0~s|JaD z(I5*3i7|SOL3H^PL8lYLb`gECNMN~0K&eRZ9+9A3A_J>M2J90VY!ew|5eaD(8PXvV z2BGM~4v7psATqpDCr;QIGXQhD&lQ$n7Dw|F`CU1Q=RF;y* z@4kCXQjx16J4+`Y3=!iTqh7upEXKKyi~RgEk$EqQyzsin{C9)o7kN5a@N|&8xK<|% zX9ml^eH$#X>x1Q`7lLKcw*m4?Z-CU$t}j0mC`&#bB)=LOC~-1KUU_+tEIlzm{(b0R zS@z-pd9`JLsjsQh(DPk^d&o36vu@NiR9`^kSN>7b6s=7^EQ1Af{RLOQK#T zQpR_50aDy$kfLh_DH$lzM%>x>Z$qDhES_qRQgrWEB8y0KBQ%c^V;pT`OeC!k(u@+x z4L68)s9rj$SI$JjXBfn63K9=tjSZ9m4x^iRW}#1yqkAtIr1N#XEGMlh`Yj&08dBi2 z3Y$baUk6CeFrA#kosR!S&-L=(FRap+;1@iabiZG%Hw_Dxoa;LIcF1;paUAV> zK__86E9}uxK-?kJd6P&sWila+%hd33rTf^r!QW5%)2fPn#cae^UPZ(MUPzeM(MzHC|3#dP*7= z>m-(EmEFP8OuWCgKP6|5Opvd>{1-X<{REky%62VSwvwLr`v4h%JA!b9M}x#cIZBDw zj*P&66Je$xvw8jp)z`O!Bphk%3KG+SAYXfQ1_VmZV(OR~EJhpttwB;SJy;4S220)? zk)5PbiQUa0oubRZQrsCVd+;kc5G>os%Z&a5W2tgQkkm6Kt8v@GJSRwEh_eVv$a6k& zDcI72WF_*Cl+pV=J2(!}#Y^ z(>GlrHeIls!d=SP%-N-rW2DhJB#5P|8NsoBv2w#Yw8jBau>jvuNI?{WI=TGaUglA^zofk;EqhH}! zC9=vXvRB0;UpvzK1K~f@Ni(`f`R5yC9C@2A80419bH$?aE`dA;UwvpvIvugJv~Vp z_1J%wLB7P_Jj@^$X;0H+oiyBKyQGsZA88xOKNo4fiJLOnOd@^6YZz#dV%i}uiu|#|Jo-zmS;m2Ct`oUM81IQ-@iNce zLoY2R+)F{y8Nhg7kIXadk^6{UnpF>xO!&KD})c#)N=A84;cWLO0A3`D>Pm;m>PYn(`( zAe1n{;bP3 D|n#WHsG>0~TvneS4*ne+?m>L|I%TMN1k=#_X+MabYp86b{!_i$vby@m<+%#LgmXin}D1^xGBhKNYftD z97}sRRr`?tCGu<>!n`{~Dmv-YL#%6rEir}3w&E~Rb~}rBGw_>-Y&c3>(p`i*1`6q? zrN~_JjN!RCkh0vSPpS;E692{cZCgM;U&p?17ZYwFergTE^KCk@M84FiE5r zY3jfRX}7;AzgC?rB~GcuKc03HX9eye%CQ%Hcd1AM?#qN#w*L+KSMV43Psp7|TBKnD zqeUk(=%<1-ktCkwQMRp`$UD@dn6@q=YzgI0<(U_@kw+zScEtjnY^3bg;uwT<} zFY0AA<-e->jI^@R+i3$;$3DUo;%7cdU*n$&-l708Ws)|$qnN-^*BzBnrNMHO^1YM9SUM9VMabft!SXBO zl(2ShJHtFkdUpsjj`aQ)S-OvO=8z63J3dGrIizz8c?!(s><{vTMePZ^@6zw*1Et|I zWv8A0gLGHHZ(%(IfF&_l&JxetVvwr1VChAF6aOp7J78drzJUzJ&urs-fZR4dNJ5m1 zC0)vulR}%Y56IE6KcSs=<5!4uC^?M%flVh?bnnZQhdrK&els4WEs<*eH{PZ1x^z-T zm~E^V#nVOJCVe%=)S6l_nfAgjFN-q@<;lq>F1k^zEzB3g@$=39J=g^CieRXcA+#s` zT*;WW{`#0LtJYFX+BkPyZFZe4vav^P+PWItHiS!Ea`#s@Dp^t+~`oBKuoyn||tKdX~}C*9ZJm`W=^CnpG#OqfcG$k*uR?bPW}=B8Hq z11j@5=Mg?9k@<&kDvvg~9Unk#HLGlmM`WHAx*wTqQ38LPn&Imb$ThM!vXH`0%6P~h~XmjHkk@3WN z8r0g8JC8AoRBKSd3Y}b`t@4n)$WG3b-fx+6b)%(tlTPm7R}#k>6R(pS=%(&5V$vd{ zj66&$B4nE>LJH?ZP=ESu4QoSegqSQ5GKsu$t9ee?Z70XbPjDAHnRLr}z6)rl+z#CN z=*1Coh4j=u%SN4xd42=8T2sTQ(;akU2>o?e#K1gkJ`*A4O=HAN-)TIzO^=Yu_%TvR zeh%X25w;a+#(ygL+DJbyjlLcmAq`Puq=7sud487qj3WL~@HW!l^`qrC_%~3WHAoBo z!ztom7zd-k*h>8&XDsc=o-L1dnS_5qF}B2*D8hz|FYDwKX>YwtyIo+Or#uG2lynn^ zHZNrkF%j0N*oO{-$!{2_HW*ADi_3Mg+r+a@5(0JC()lfRaRHm)x#=2qYfe8jFel9q~`-ko^zZ*wh*_Fv-K(bkHH8v#ER5d z;@r_4=(DTQs-4g4+Hk)^YO<$1yt9SW8@-0!ewxx%yEl)vyo zuq0Dv6YIAT<`ORFHuoRg2dF);kvjx677BWTB#*dxjKy5Szr&ipbr<`GO{{l&xa*Un->Jj zE~J65=8j+)j2uNV{+r?rMt*9c>;dE-NEqBz@xk(Xs2CZZ#v@E?W7x0Qk_bGR_Km`( zSJ21o#g+ZH5U&;r(TlL{Vy8~t>Cj2ZL`~XBZ;I;wx3J%@xF=vds6E&JP?ns##2+a# zo#%y&o1dz1&tVtO2FPum?@iKVCVtykUpl#e@nZiC3ple;40Ru&?twnurOBII)BOU! zyqnbfY2rTJqFodY>+0xsPn*D{8K?Ak8eotBFyAjaXz;8 zO|W?OakeBs9c9@@qjr8Xgt0qBirH5b#c}RoOuPg6PWJUsSk4`w3Hv7e^Y9XU@CDfOq@af#Ixb(VX%BGZLwBlM`DP)Cwlpae%@oz$%nUSPI@Fj1vZKK67He#+pGLr-Ap%*MRC@8ZaQ4ZMr!BsHR7N~0#@5fdv%{!6IWzjsQwO-;$KUgAps$@9t}>8s zl0N>sX)7mVZ354m=~E+hFi{`x)7UfhFm~d;9qQ|+T>8(LNgp&IuOqv#GvZGnuIgti zx{*FtYky9bNEy#}^iVe1tBhx(2&e9r)E(0}^fJcSj>+6B@Jxl7bU&R++|!iK*vKJl z{&XQf;`MN6$FrRJN>E=T54Cf~X=k@5L0{aVldgf?5svnB9jnrbQOtpp=;w~4$~ zqo*o2`Kx=g9LC6U+?}kw3CLtfha&d)dr8B0K5J+H%emmMq+P&1U?>2V?ePuifPW(;Wq2r&M6u>dS-O1FoOc%iX!CljLdhzWmDc26x)MJ$4 zIGh6C`JH{Y+NY{JG&Av4TN~NmsdHNA8phBL(il3B`_#w8`)VlXe8$D2A$4wlaE2KdI;*Ob?|m^Bs7(pR1DGJfc*UUctQq;r)$x$+nM$+MI9N}b#( zbiP9RYsh;o{m1iF*xHU=%q^eI79;jT6?tevX}_ZH#a#?QuBhWteOa1Z)o{9s29 z^AdY~H8+$U;v7%>bEH``);}IEs{Gl5sdqG;jntbnyoo(S8Eun4kb7}tA7O4GPb2Rk zgOOoifIGM=X|uuT!@=8vy|DLC3#z(vQR9CE{wl3}#=#x*XXzUwYrlyFPSsnTiw&fa zzl^(9p5Fi(#aGWlW$*M&-evBn&I_5@x|6ufcgc$|iBeciYdYML@5377}3s- zcd_cM-uV{(-1&EM_wT)gJi`7rHb!#q>e--hmbO zIjzzo4Yl6+>XE`bE&N=>D<#i&8Ea+Tv*kTqfRt129p$s-BG2EaUTQ8Wqpk-D+lQaB z-(%=IZnEb_`tGf|DbEGw1fKV*Fqg%j5BaI^9>QHw{<8E*_~M7TtK~y;hY;gsUKs)8_Aw)I%%;tE?|GRUWH>$coSLqLZHk* zZoNiWQ0qw1c+yBD-g@elL4MC7IhRf)+}ohmq34iZ?%G!?$$Y5hq5|Tnb6N@IEy#OT z79x$@3l!Yt`v4V=wWtW2DMeoZO5dscUnG5OKm{qvf z`Q`}u61c%OAV5AxTE7e6eGO#`>9prmz7<%bmp|h^L)ts2_m}A9W0_~LhhGfioyiC>&UzeM zMXEF=qdUJcNF!%wmByJ@_3|=)D(;zANQ3+*;69(hy#{CRc8#_{o;T_x3xCtcysN&d zmrsvyZ~dC@j4dT9e4bw2e@Genr{C-4{CC{jcnwm1f;&b09On&EL7Ym;@E-fjU*qRK zz&AU{oyaxFRCp7V{dn7B#Lb4?guDH%A<}`~_T3P1y)r~vkVW{{LUYRysYg1wOx*vm zK^z|sk)603o;OGxONz>5a?1O67MdVzMO4Njm}B0r@KEk%r#>CBZ;26>Ni zI+43paMq9I%#V$In8Y0b@*~)ruaiG;wk!NdFK(O2`xeH32mSvX_io!}h__`X>)TA( zTh2Rv#?>Ceesq&Nmj<2Gy``7aUiG3cyhd~mvX`n!Ck<7c%F7~Fz(g*`Z#_M zo{vT!3Xg)yvtS$fV?!#ha_s(nZ0<>(HD01!NmG^gX%+TMy)=GDnN(Rm*2`1({dN(* znP=S3$usyp3-2%vo1eqpXwTx_Zk9YaN(KXw+ghj^5EuvS{ z!?-^h$$boN4;+NUV612DDPny@&t;v;dxg33wE#JeJCFC_#%|Wr$?Q*M32`dF~C_qquq2LAveW zmdC|{?N)>sQMn5in$QxiLUM=gm4eAJEABp?ZQF`0gPv~pN*E*i~FO3xUi%-Zc z!n?TvZan{l-uY#?zUkR9`ab&X4*oqKbKg1h34JefRvV_#v1+v5{oQE2M~3Q415WAQ z{pNpk=U>ptdjq2N_Xux@$ab_=(c_PYhB3?S9HIBwM19nH9{J6 zYh?fSWbR;-rEx*BILlLI5^0}buaharX)pt3gAqGY_esh=V{p%heHOiZ!hN`TPOyB+ zw{w=RQBu)8N-7ULAyvHhtL}-AngtPJ)eV)}o5RIs8!mOaN5y`7gj7upl4_`l4HE0T zAgM*#IJ4Jn4-$Jm_xVQb{BeUUB%ejF1eU^bSP3oMVVp9_-xn}Wpwjoi6Ap=kx*D8}*5g7$VC2$BTSnhQTexxa*lG8t!su ze*k-gMklt}fsG=YiC0N_Rc#E7XE*0rsATy+huU;WC48vmplN!hBc=i=gE)bK6b6NxsWEd!6<#rTxLp-06Y( z9^MbaesDk|IIoE`onftk-XX!#^CUdW96lgemXPLBSPm;80aimYq(a{Z!IF;LsPrPn zbg^DGqi2KL_=MyloA7f&A-XG(zJsmkCE#I?;l|xT-?c+?RFE`*6B@_z9qx2&1sWK) z^#gfd##nYQ4w4RJJGSXc!6r5ZNo!h=v><)wXB+c@g|&zDcf(#VLnYV=SBDMRh*t|% z6ZWu+F<%-aRmgFKDFZ8*4~2^zX+c(i1HC}7fA&{36UT@ZYG;iR8`PoO$Bz*QdNbi% z;DLkCba#w6;jr>QGe+9@Uf>w|aquKXh&w()I*{!L#z@umF;WdRV1?T0q|ZD3I;4H_ z7&%3p({L8fK`&f{E6@im7b2wfT7=`Y6}yG=mpFL zkh6^OwQGz76USLRMw%A$4FGx=42QPK*y(us96Rnny1}FIJ};K~Q??q)XhqiIw(Y@2 z*+iI%)|Juw=n)TZfW5q?Wx zDJ+MTkN~S88UES+Q*m#E%}~hRz2LO~$$Ke4vT^4^UO=GqKFNHC-cyCmAp3SaAtlH% z*a^E~FPNbc9Q&9zkand*Gc;wf{{R=dYZ?2G#Rl=99|RBcgc~}bow=rJv0kdFQw{Y| zb**($w+`xvwA(m~R|U&q;vIwIpuQbHg)}17`-&FqxfR+5(!bC@Kcjn~zKeIf-F!cV zbR^OK%*Re>LT`@en}inlGD3Q1!jMsNnzYZtIk*T{pbu`qEpSH;lsn2VZh=IA#mlrPckv;4S7b1Iq2;id_+AZ9_ ze_4cI)ph0~s0m?i3S^D~+d$?l=0iL4;1a?t1)u#5HOO-Gm5>0dp=B6z;YiBJ-06b$ zo9qj?!*C-#m)U>BGMC0Nr>y)d5a?zH%aqk7+x{YnNm(zyK z|5f4Cf4qN>Vnxx4Siues$Z2Iy-oU*(vgI1%FQ4(Zo$-fk--G=l z-MBr-deUnMq5ib1;{xN4es?09pt%eCw**QvX}Z7zhoNF$pje!Natz&iG*D`82TB1S zeix2rO&*JF%)&;dVk;*Dum_*H;wTN6Yw+NG{_|66v*Q|Jp);Ry;&|8WH!1LJFLm%ynKZAdnQEe zv|Szbw8c?Z=y{pnx)84SQGTED*$|nFJ|7msB3J@TVL7aX1XvAyZ;cZ7_hFKZo(k#U zpwBlVoxA8?^v%j2+s;O|F&130jE^|R#|p*=(zAg6pA#z6c%BP|uoWusv&4l;3A&Yc z_BFhBFGJr6cHZCD?FyxjLZud|zBO0xrEmx2{UP%2B!8rXx-}x5PVy)1W~3u9 zRBn;Z9k>UEIoMDLVIUaYHITG*p)wdf3_NL};>O*9fBUdd$)WvKANt;dCi3kYeKmq` zqrie*Gmm=3^4-OY>P|wW4!3jb<1}pWf z0b49}1UqG{gQ_m--_5hhod3qNCPuL~BI|J5k*UN@hmEiqvY`;RLJ73wGyfJd|L(!Q z%IW_G`XAhzIDbI>3ie+Y*njn~|GLfmdztyS8~f~H{_SAx{}$UKo!&?J9tQc~T)7)P z$Hv{hu&!SXl)bpkPzhG(<6|~E(gDqo|5}!~kRIq=#qSA`J&ig!jO_b3RE{AJGH2#} z&zy~Z3QmKAH76H&7S2I0G~wrD?Y)TZ+Q%Bk`g#Su4?LV1+&0!OXm|2mJ>#$MS=RM9 z<^jf^`!$2yAY8%sjJLJ?ZVUYm+yleUIWI!*3js11*|UxFyCq1%(1*ha7zGvhSx7qq z-O5_1&MV{4CxD%@)JS#?2@Oc;laaf(cbrfSvzaoB|0ebijzidgH)ms~UYZ*CW(+Dx!wPnl-UY%SEy#)ko zpGNtz^m34Ahv69HJ^wZ@Eg6U5^iLq~NAYuCeOpc=3tr6PPL90^`Z?%@i%_+P`X9pn z50F08qT4!2|2pZz6~gqv4Y&nI)&2u{4-9ju{{ieD+Q8LKyLQs9H?jY_wDTVNe-8E! z`@sQ?;Djb{atcg? z8891q7(X${-WT|d7*gHc8llffo6O>y8T=N(aO`mjvZp>xaY!hC}?54 zaX-HjJpoojGNeK}^ga2uY(zSWvt%_xWlTa_%?iT`fc3udT@%@PY#qFeK~NDWxg?F#fdu$6C-T2S+kW9pXv`KRf6w&d|% z!a-Q)Sn}@77T3va|2+Dox|hplUe0Epkj*|JTdFQ(OZClcskxgiR%Gq9Y_UxzkEktT zM>dm=3p{WT4#P1x4yT~y%ob_wAs=u-`vvNDnL6&;A|9xZ-NNs(x5$2QK;z9V;=H{@ znh4VjX3BG#bk4##=!FXB7O^B|%SCi+IrT5zB3ID+z)rd9sIQGSsD&H&sruhS-hrk= zI=P25yg>a=GA@rYE+H8AU~n@&Jq?Ug{5qhjJB#{fQU5INU$Ue&obj(?{Lfmtot`7>oT+#r~lg z+}tmj)3SW;Ak_L_k*}8tq(2EPrPyULb_oT%*DV~vn!Vj`m$mrWU<%>#j_}=)H<0s< zL8jq$BAc!mWCr?dh=H~g&IgI?8?u<6kZw@l1kdH!eDJW&n#Z%xK(`<(CUduez66#+ zjg4|xm>)XG7g+~teq7G8m5>0dAsKqT21!M#d(?DfALpix$jy)qxljmOp=W)zBq;j} zz(#QIRDPT{b|ZUV)$u}?`3!Q#W5eHUk-hkvq3`?_sYE(->|2lpmpF?)6Cif7-0x>{zprt>AI|+gvSl*w2d1+QPh}kr zVci|cx(n`stizyw>wlPhkHK*`1r?dBfAfOmG`h8c^Y^}BDLI-VXK~xFbN(*n{Jo3w zH~!__nX=-e4Y@vQINf$~0WvvV=u*R9wjh=wlhP$Js{sMQkmq*He-Q(iaJtd7oFa@fH1xj@|cMP)v#R|2P1I0%B)h!5Qe;X*%2r~m_ zLk!G?`LGZcLCXr}pCsm=1K2mT-^RYdcTRA!{}{+`OksZy^A9v;GXG>T{~()F@M{Q= zJl^5D143j8=`Dq(lL6wi1jur9S7!igNPw(FPXJFVX*ZC^bn=4L_?d48NHQ`ND#`<7 zBXToXdIFfg0+_#;yQ)H@z&k|hZgQ`2m-ifLyypOQ{_lB?-?l!_yNFl$o)_CRb<;i?Ae~O$DIu+(-Z8naa_&ZVm9iG>V$NU8 z++WNZfOIco{=XI?)ptXr<|gO#6wZa2oC~u!|03p_9XWJReI^gZ68V5IQQC+wFfx&#?oG#J#xUYI7}`Q#!0?Sa0R{KJo}gc&ZjH5 zADB$}rc=IIln>p*?=q_5IRCF8k8;k*HqvyGb_aPN?YR3$;|AP!whdfZrsT0@Gjy%!U}43%>J;M*FZQo{xJWEP_0~Pf+ti z@1tR|6nD@05bS_E`Fh6cqujINw-ORyH6%kS^nDg4>Bx<+8L}Z43c;!cO?}B#5e+T2=eVO&Qhi?;kwinD$30AO!1Dc^Kjs5v1wGZH)A8L1DH&BOe zhk9(?cfS|Qo^Kxadvmy7SU~>RRx`AW<$eO%zyjC#r-bWnayFK)t6TDAF9*0wK8qPuuWwV0s{{404tpn@}jW^XOcF^J0BpkaeMaB?;VU$Xospm*7tD#8Phu1f2OuYvA$1deV-U4 z2Psny^S$Xpuoz()X~aOmXzZ$<_8rgp_YmjbPVTvp?cJPzk#5``Wc@PEzp$UQ9MCw6 z^Y3`hztBv+<~{sglJw@oLRbV99jyPb1l`K>8d!?H9PB-m?Jj!|sD+jIIVfKOay2wv z<31mm3Qo$~ww?8@ly|ncH-bBpeMc7i5EJVgvWjw4Q?44yWrbRF8`N#0{+U6tnK;>y z3x%*1N}vpOLQ5C*@238g%SCzGFHrw$)E}wtO6o(X|1jzg4*VN$Qh&lVA)9fV8@NN^ zd}hK%)i<_FXzz+@>c5Y72BdE#{hFgevKQUGhVwsVteZ+1Rap-ONhSV{>wIgp#(zK1 zw1WL0es*xi(f13Om%xF$8Qf9KOJnK#S&V_{Jj2fedCZY+<7hdE-orijVPvmRhXFs4 zW9Y|0tv9ETr{OG|gI>4@X4cdzNXr?$^dWCR#Yz6}2J#MA4`IV#YsEI4*a-V+b;nqp z5-3&6So=eG&NFqkS7#7=AkXo0uL+lX#5cT%U4ZYq?t$3rF5a)=ZtYvo z&x<@pz$l1-aWDZU!4zm=pWQl;b|$=wc6^xR*k&x}e`GU# zQSfm%-*@m1mbCi#W^x8{HpD3R=lqs-?P%%Y_X%@x_ttYCg;eK*h06T`V`?Pl6!ayq z6!Nb=!`ez7><5Ln>5gW5jYZT+-34 zfxMgMef&oB&0rtM`)?iZsFAgVQ{ViWK%EP-d7cY}3f|Qfyv=%mUIJyX6L!O1FheC& zSt6v`#yfVTm3Qv7NE_}tq+P?0Fm`Z2Gq}J52jMWZgz)}h0#_y4iH|BvGRKmSj_!8ZoSNar}5g4581pA*iayPSN#0O!zq z!Na?Kcm5dZC>k+m3X9Odm1+ZxH_$6b!}AGx$Fa==Y!{OWi+l#se3$ zLkD$tU#I?1zlZTRkNYRaoCDdog0X>Y!rhECEWoZ{FoeNy=;IsAzN=$o1bWZ6;nI7a z_3PO&(vzo?QTRo`IG6wqY-k5}Vw2FPz%)>HrN-Y3bQkv3#u%NA9s?fS?lasGU~lcn zx%kb8g{kGHwoM^|@%r;)uM^P3XnIi*8i zBj2wguP7aEfHRP9?~tdU3H=uG4md1S=FP`;QR0LcBh>0eNeO$Svd@xCbrRb1Qb;2I^bCcI>@Fqy2|beyBgn{U_{au5&;m zx)YiR*PMy{zs1@A1I}Umf1w`!zkp#O_636>42Hu97zGh+ z|9pNfB>q+?0e1#Af$abO#zg-^dp`ZYo&LwR)IEFsGVBlbUsJY*{oTd>kPp7U*-0Ae zot^sTcQ?9YL8$CS7R+S6;rEE1siZ|YI>wSV(nZ?pTOS8w&rCS;$WW<7TEPwusJPAd zztM<^?e;ChmptNIGlpha2C!%FI;Qh2IfL9ze}2r?3u|P80m;J$RgwtSPILbDIEKUmFTX4*#9j4XA$~p=>PujHhZOE z*uM$;$7X8wVE+x+Mi(~Hjcp+9oye*W~Y{&ldvHukMpJ)Bw{}HC(N!n)sb1-S;LLqF05>R%o>fiGk-w5FD zrM-6|cf(#VE8=*EinM|q9N_zYGn_My@*hauW^q13_dx&mf4AxXVc35X>kp_k+6uMk zHmKXfI#nJd>BKn*hv66;hf{DG&O!_4*w)+Z1-5hk&1X#{UI)?*9w+DDYn*=xv;Qt< z6=WlB=Sj}L&&aE)%AQH27_tYbt3^zIA9u%C@3_P=C^HrmT*E?2Slg987>o z(4?dO_)kH1Y1AL4q0ayhedJD~{(;mVX5(j$Nh@lMf`@5GjlF#{X|dPJl@;1uAA4#4?rlpXgTRuo`f$ik8{9)tpub>Klhzs0xpk>haN1 zGdWtU$l9oAu_5bl+mQz;Q_f9(9{vST{ZQBTChP$;COLR z&&@o`hNg+pl8Y<^RnE37>NPJ~w&E@UH|6qlkT><}K<>?&DSt?uC3~*Vl#doaC%czT zmk$?DmtEJNl@CIu%Fe{8^8SHmrF_|B*%3Qg%6guW_s&d|(q$9n-RnED0YuuTiU=_ibtJ>DnbQL>L8mVTTTDW4wVTirPUQlT3l zl|2Jx3H`P2CckIA%)LQ3@6-6rm+jCnsXN0v!&yV6Dsg~R>xM|p8Q%8{43T%H{zSI! zn)6-)4`I^ic`aH#D#jKhTHZ_gnUrmsAy)kR9*>syV`t0G))?N|&X)2O z|0>mcqNJ)gO02C>Qe%svPokuDUle!fQM})b68EJ5u@lz;&ENtL9E8Krx-3ds7Dw^! zB}&>7qj>ic#lA2~JhYJ;*>UO}-)qdpYAxh0N&VikZ(Fo~{r@3a!g$_0I$A~`)&1Wn z<$jWV|4oBLpcm3k1#1Ij9D2nv?0>OACZJn6JQh4zF;c|2r=V|r*;Y#!b@NlMfm*z^^+T$6 z3w1YH|2fAQ*(aIU1LhoJPq&Km6#rAPQ231sX>Em;lkES_u>b2}{|D`*?EhG|+~w^5 zidp~J^EG6$|IcF2j%>v3WN+RC&ER0Y&Ly1^D1)8Ql*Rgw|88_w18GnJ3QogWI0wCO5!AQu>Nlu;{4PFclgJhP`XKKJ`&-8MEp#Wc3H9%Ut zUkhWu0~^@KGw_r%7KhQ!^soAs(Wc64p)UAU&6_3FX|v=G>D>dv%h(AFhAi_4X{0#=WRa?MJPfMim-$wEMW;tSjLN4!VrexVk|I1iia?SU_uCHFlkFj zYPDLT)%w*^cPQdwD8|LG48^4gMN|rxP}r+bgd!Bd?B_IwIFrnsKlYDrJGz!X zobp7+O*xAF3bdVZMv{P!yMZ-V_x)_1Xs$p+yhx%H}j20yS*Ah|+%Yq1_1u^DaW z+5eZ>|77Mm`~NQcA6a@1jhET~FWKX8CR=}LacCyfbJ_o7t8`PN+2+#Oj!NuAi@5Y; z>ksIe9nv1k{-N(j?hR>ga}Kg|PjNUXF1DP_97$BukCJgzt&^{%@^!p8B!yFGc$fYj z```5@$eOe4|BH^B%Kwh~vC0J+l;LRoZ>#@gki!Y&{Q>NhXbtd5;Wo##JNEbfH~x2J zB#Rsxm9wVXANk+ZAObk2B~)M%ul@=%Jezm3y9VLKU)8racvzVy<0=Q zW3Df^$?*fV@#>IEj=iOfTp@cjz0g3b^!xWj6sO@`qJ$S zTQNj<7)nt7wsz+o_7Xk6KH(Sg|E~O>mw&{s%Kv@&kID$5vNFc8<1hh}Fa^_4iWz8A zp4*l2j;rebUCQRZkG228jmlZ{U8ZA5q4~D*e_#10Tm3J!)EI(v=3pKcAo9(pH<

Iabt*H@3c-%mZTQQVB^zP{&O z#Ae|#l>bb79DWnF(?5RyDus7qH?nNw{p3L$Mhq3duVlN^@s{)4!96@c%SiRG`0}rg2pQ!&THDZX zfp-i!Wjiab3!P2&5u58hL+@K`w58VT{M!BayAk?6BSNv`2Vn??p#;&`+$gfgy&OZ1 z!vsvi6ih=YW}sI;zx;jU|KIkG2={!I&y$>o1z3b=%ybF43@fk-(b|bB@vG^N)^D$+ zXKRLsf4hFWW}@}ys6!m}ON~p!q)pGVZHE1<%(>*_b%lC7dO5i><$(ZvA~<>+g%KzbBhV7+W`gpoQLgUtIe$VYBqgupO0X z@!zzu@tyRH@$!zl&xGCdy~z13d(k=0Iv2TLTueF#$-}7LW;~TVimE;G)g@m$Ae=roW>cnU6=pcuJOM7lbs*9 zHs{SEhsF`|e^vg)r^sfVb2;RS{^fRHyaW_y+-|KzurN`x^&h;cV$>X|*d21l^ejNV!)Hdb7 zacS}nc@Hi0?;3A?rEeJJxG@-q37CW_ zn1=T2-ob8V`>rxhb_sWqIdwsnY#M9+&uD$`>H6R!m23UqW_{n*i)@sE)*VV?24*3) zL;r%TBC9cvz6eWDC*Al}_0N6vk8~T5JkMUAt*u$8KZuw*A^W33{uuoNEc3q#`|edD zzrIl2T|ZDips)6&NS#csaLg*KM(@vk8;V?yjZyf!zDJ;*?0#Pv5$+lOOeiBG`+s{B z{(|~T-gbQRe}qcmo!E^Q^?Dk6>6!GyzUhAYK}0rbcANHtjMj=T5O)|c97P;Sq|l0* z0`t02x4}Gbws*a@IH65$sMS9}qqe?@4R-9V_Wic@{ebozEpKSwx3R;>c66-KKU=BK zu|?m5%nIkoXl!Nc0lzs{G)FZ@p1?_*Mom%QP`kzbn`@5P5$$u8_E{e#KR%(YZrs6+ z+rzG_W!LG$H2Xd6_ZG5slRZ*qvjfqwn13Ff^e#NwpD1d7&p1!Tv(~yjr|(T~IV25S zpl9CjU+G?=UqSA^|02Kt!7etNxY$sA81e?HFB*R!Z=s4_cUQmSJ$)JBduV89@1p*c ze7wv4)Q7A^-7foyAF>}Dndq`kH*Fu(%gUbr^jWBkVE~FT2tzOoZN`7w(ZLqapp)K( z>}6%Z``K8l|BGYNOi8oZy-uTLvHmZz&l!J{&M1t*I84A4OhdlEIobO?-F{QT72jt^ z{>r=^`Yc3qZs(BmumFpY-+x2=5_)F5a*1X1708WHE)mVa?bJ5M-_y2ZmETq)nj7DM z#BpWq6#rC}xjW+5Vv`kvo17nYyNb2b?D9kGa`t@#J=tCy<~YYZEWjcx!7{ACDzxzdv>)Ij zIKuyc&iQ-;{0!OI{10ek<2SMOkMU8Y$Y$Ym6aNER5v{*pEuFPkkBw+?o$0|(h0XNL zKzZxWM?o)tn@zPr-h{gx(@DOrtmW6=D@)P$cdBRlUAw*A@m20$-CXx|q4oF1CZqX7 z4Z=zHq@GMTw$gDsu^W4_9|tkXeLqarI7cn&hDrzZ^u%({2a$i`v46okmAcKo@ch%B z>!bN4f6_m7Q@KYpKQQJTM-fL7DMaJ@tz=}+M|=83^8+%%Ih?>roW>cXZWe`e9q zN59|={ftG*1d`h92H}LhNwfxVtGX@Phv)UV)-?IO`&*wfR;;h_#DEa3zbh8kYv01& z&wM%zqE~#?zHndm?GpMhZ8 z87DLSm1%W(et$N4uE6tA4>YMGj*%(HHNWinEcJXAvj67#Mxb;`F$1$O2lLSTJe%UD zPlaggzuP9bJtkT9y!BLA;J4SKc5{nzk43^uko}qVcc%5a^fcKrmoI}}@kROlzBNqV zi;Oy=^S%eqJ4&TidvTT#FX%@_KS3 zGW)cn=vv4}fE=>e?6)%f(fGnP?PS`Ra*chu{a%UK=)PemSw&WlH$RQO7yEG#hY`b3 z#F0b_t;isUSZ&|%le2~4$VKZMSNeYO$Pr=t^}cw_}+V;47)}Q3_n<26!vZ?vJYhl7ySOicl%q5SQy^i^`!Bw z{^7?jS^tNMe|iwi?Z|%{ck3p5u#PMYud6SPysQ22zgWTV!cQ);=Lg>p@1*aCqc-iV zF8Hmo|JzWr@2>vwZ9dxDA<=Y`ul(2E&0q7;-V9CMAB4t&_tmYx3dioh@7=o2$8bGd zk!BCB;Rbqf3wLl2576i9-v3>Np>|(ks6)J}Fw}PyhJ<_5aJMidZx-r*|1R`%+yE3~ z5QbnFO3=2cuk{~&t^ep7GGwPQ$1XA}ocqAu4WHFB-{1xcNzatUS3yT z7D#gumS7oHpv|%Ej_n{b34NNhzD<`t4w-Y_Mtzv3^ZXM?`Mp{Eqxk`?!mFgQ8nLtb zIb_xCK4Cq%5!EAn=Y%XnoG+uUuWMW98vDC8vSAZ@>)C!`yWe|$shpGD&)SPB3R^3( zo7{{2=>59=dC0?vMd5$4c7&{%d!K*heyBTpKg3ag^S*Jh`=J3z%y3M!Hfq~Y_08D_ zAugOm`FG?IZ5uvR|9zm~NN>6yGW5*C`=J9l`U&I?NPCZS%)IXz-VZ0m#YWr@ zr^z#@o_s%?BQKz8x_k|kFI*D7f`;=ShNNq(FOW~vy!Rl~%2!>Kw~^wJKm(HK@tkJkUi>3RF)fHF#t)-2Yk4^s42QO)sHNHyU-ZngK$B(r(9b>UWs&ELrR-*gWU1YLqjil3(?qe`Yzjoo~hC|am+pX z1LO{Q56~sPGok-)%(b?NojUO0xNH5y$p6j09;^KwZ7)0jdHt02f5-cUt;Wmy{BPb< zj&DcAD~A*`Dp*268b14{V&>kAR#{5i(rhnlV~W|C`}IvoHtqumFp&1k2EN z{ZHh74T_NV=B|Qy)H~dNS{SnyvrK{)pE1Y<5h3eGk2iUhzfs>bLj@#_Io#_ROby_9MLm zi1r`d?wCrH7b-8|$3I#={~6)5#RfLJ`Vlo-SqB%@PC%pGn%C~kosq9C4X#9u%KlW?R|4-l~PU8&D;R62u`v3Clf6lu$ zJYN5!@17t38LzxK#}!<|4fNs`?%*EU+{bqOQU6+hJ9kz8Uq8QTss2Bil3w#_2Gz-vHqr-wrM&Bmdu^DEy>%O8q&6 zUil^a3w*^~1$xUV=eW#vp=Xr+j~qx96KJSqpQG_2`}za+^j-PqowNE4>)-hh=FsB@*uHzL5uh(X@;=-5CL8!JTX@@3VUf7#ycNH) zPjRO@-L)(cw+!j4`~sWUu5-;@5MG7sWPX9s`~o=XdUU{*6j;JFy#ku^$I<7%?1097&{*9}hq8d0*y#6RvrQ|8G9u z4B{*KX14Ir?9ommiHu`%$hY6x@Cxl!Wx2Tr;!dN@G41Gxjzwpa`-?21yfBOt24-OnBHv3?2hXEtl=EorzykUr|Bby0e(7DT<;z3l(@X609v$!=q0%usQLQf6P3}dsCgk<|#@xxR z4$3^o4@Mz!u>>)mYeg5P3xbr2ELMx)RdFk_xr)Ng`um9)hCy*PfP88Qg zc22f8v$&Wx|1^09)%0`Z1yt>E-Mb!M_a)&g$X~a-)UT5dvSzUIx6OS+9Q8=Nq5f}j zU(w?~*Kh;9xP?2ohX-iW?{4RJ?I1J!F`ZuY5w;|DL_T=_izlVPpwXv(@e77>vUNw1`V% z5Hgn#|983n``rI+?mu!}?tlNt2GI9+T&i3B zhcsHQs{bzfPO-FhV=q$btNmp9GXI(Wz(M+9#Lyux(OTwC?RVEy_sxCHp?a6~H-3+! zYKMMJbUd3VMYf`0o%aa!YrI#>qw=YrfsEgBIDzhR`z???pEoxm3cKfLqA(jH`>PS* z9Q^`%`H;(hXKsNx2A87nm-~e)(Q&wj8`$>Id(=yh)ZG2 zv!lBtl+tG)cS_pgy2N!l&n$7V<0WAZIS}qs^lwC5|#)rLjygzOFnDm z({DA`39W&gXg`hORdv2Oiq{L%d1 zxyt`S<$tO2Pv+Nu-_-v@e*Uj9_|?)|i}l!umY2;DSZ>Uhp4lVqH}vJ{+mWCDE8Mls zxyZ^W-ZP3#HD7{WHP&1Saxbb!njb+PMBFvj9rq5N@-C9eE$TS(u-{@h8XbcqQfNg6 zIh?>roW>bkz$HX|hVEY)qc-N>^Q)2&t-FtWo7)hLi(GNcqxFA1^lP|*-iXh6x5zuV zhX+xdbowmwZejq65zYS}L=M3)lpsHchU_s8nNm-Tp^w7^OhSveXb+$%^!)haX!{J) zOOexO$>P!YW6fkX$Ts~H{HgiBTkrpV{`V~Zui#6sv`^3+dil2tL)&c6A03N5e{|Bj zklmzyKz{yjt+@wXt~G7$0lO!S7QaRFf9FYO0Ty8imSF`}q4!^&$*+It{)Tr%xX0Y{ zwd8ti#AZbEf6K`2sKib@n*Y0-9v?4nd@FnD`;i>X?m}WDJ8Uf93){Dr?OV5y{kxL= zyM=8@=GVzB*CzPSK^#U5`TnbZW3>P6QQ^G(%LeYa`;q<6?(I7CG5fch{flEr-Dm%@ zdDCp)XrKJni{dUjuXK_~p%u|@>D$tyXGTjK75ZV7e1tiEiR?({AUj=q&M}b>;RJaS z)koB!Ea@DV&sf1dB)Gxp^4ze$*ao`13T2ia|XK`A){z0W!Bm!rchdbAgJV*W#4 zUvj1Qu|Rq3@10%jd<*p{-Y*Js95)XOko|2@SVS(tGUUG_pudn^&gX}=5BTHR6_55C zi@t@LKF=i8-*hAFT{9&7;JqPX-`PRo&6huAkIGMlAHFvzY+v|G z-);Y8*ek78RPGuPD)xLP?3nr)dteR?I|mN;jlSU_|ZIH)?pP`cUAC(P(R{@kR1C3V>Dk- zw>)qB>v`qq*>LRs^ZdKk)SZ3STr=erH_(e)xPz#V9nCp<{UV>JaLqve4StO}a|YrI z`8W6q66^RkHt=t(DGm=D({H)w_+9nSC+#^!FUBAYL7Q>>_W9-rAcM}8<_4@WZcgS# znEx<>f5iHOV@RR-UH*~x_(#arP3ArY|1`AIk*#B;@#B zvITq>gZVHpMOn1W{|T`wN1Vb$=6HfQ3%gN!%X>-$NKs5iJEQpe?%SP zs7C?~O}_ z<~=_)Q~ReKYIbaTg!XSR-(Zn3YU!;;s-P&WCDY_avRs|hVtr!vxx%nnIHP@z=HYhT zbq@XK{5*o2+CIN+Me8(8#RrSfc+Pur^8`qZqzK`pz zaZR<8)i)HKn8bhI{mQm$`RodjZk={2$usyT)H{8-Lwr{1sc(Gu!YwJ3F<@UgnO8=FUam z5E?))#vpXeWzWxKvoB@4zr=<|4nzDl41J_uLXN^1jKc)v`zPASx|g+MsMnt5`zKL5 z&WF%=^`rTRWb_??=IeY1XjyFjA^$`hJHH(r$RL_~(B&GUxd+ksMC>>pi1SRrG?b#6 z|Do!v{eS53{?6_Ev*>e>)aEuIfqHH9$M-**=lAT7*z8z9UxbQaJ^jzj-={Cb3ar9v ztVNCXx^|6o$j_hhKdg7mMr=kIwxbd|u^Vk{igva|e*c+M>|ZwRAMAhD#Qtb!f4sr| zK(l^H8ZF+>R_|xZ82w)9?8iYIMvL}0jTrsW`*V~YM^vV>h}y7DWWVXST6^{SErn`j zy_L+Mie5KSx$@5Cgv)=S-=I81`|8&ZRSuE;?hE0h-yVI>Gg_0H&|hyDuYW=2+q?Vf z>*LCuzRKw+U9uWy=oQ~|f4^oAAo>MdLQjn{L3Y2xe?#_uLs=kiL^^Ka4({OrdcJ2| zYlY)qC=C6`0Vu{Glz)zYog9V|j6&}#{lXY>98%-;QNH9G^7JW~h8A&Yl+rV<1Nu z27MNC!dc-ivhyszVzmFyPtEPI{x919=ka`|v-awD{5&i`_MI2PB610OhZ`#}F0+iD z&wt_cu!5fQTvw6lp`LGlYeDF1u^t@->=SgN3t2?-lWKOn7;3F)uS5LZ7uAn1h6LGg z{Kb&G|6X;Mme{AZDVZXE{q5383$;rd0n(w-1yDr3pkD`H|L_HG7kAKbgejv4{ zG{hZ~w!XY&W@$*$E54}RF4P|s&Mb9(b6wwb*EiPnjVui{bLD@b^B}%bx?80ChWxiX z4^q--MFu&Xz)76O8MG}|{?LI8I?;tJa%g-{{?E%lqP-59?M0BTm4El56*2ewoOCYW z60V@S|BLotd@=OU;{#or@HP4kB-gk$*O^%7S_@tbz2a`+4({Or`g}{D1q1NdU)?u^ zV&QC?`Uykm!|(^=U&>tlWaUp8Y#@`uC5{<|F&Kvln1m^qhPLba+vs>h{}!G4zFo-D zb7)-1|F)F>jZ9rP_I_7?94++LZgJ|6Qt4DYtDXL5`|;EB`*(Hq3A5-KX?5%|$Cy43 z(YM30=tAc<`~HWgLrgugz;BCCE$t=bGE~v)&hsl?w*QatDnxT;lBj=IKFOZ1y%@TG zUTTkB_xlBVM>r-bUu((r*odfnZYHDg&)#po7|Q6C=>1aZ!~F;OdtcweHy6zF`n7z# zW&V@BgLgV+H}+yb4&pFkIEpyh>`Bm$jsx~E*kkX4c6%5gd)6KXXzc6#Df0f1sS(~E zGA-OfwmPonkoPy?9jWsEc6lergm43y++zQLdVK;UohyY_WRSxN)M(FZQ8!lmgJ?`S z+FLe1{x7`UF(+{vXK)S|a0yqCw}1DrfAQG<)y_X^-yiS)`vF_`KKqwUU)Dc3|B?O6 zUhk34HQYciTG}1cWX?Q2qc72cTl7K54P_I#uDt#4I-_zQyQ-|2Vcy{?UvyTF{E<+kvB`GX~=@0WEhObJLzb^vq@X#}xWB&R&hrnKX{GvMGtg(V<{d=dfzeU}KXpM3GaqAf! za}b9SLvJJB#Lvv>p~unls`v9vzKu5PaDL8jDlUaqWRSxNlz&OP{`I1e_m54MpXtVE z#h=C*&|g9{!(>*FWe-4nQ#mA)50& zgdBzvj6$@xaJ0wV7+48=|p=QsK76#`lpQXWzsoeVJ{F>`>$EXsj~-zt;SJX{DOX|6j+p zMYKPEE2@mGXTPhimsT}P`} zYv-R-_tT@j1=EX*!!mkigKNVI`YPnuzgcuSrgJ6xcb)vB4sq1)F}DB>NMg1Bti^h4 z#AcLXJ1WuUKDF;+|C5y-tJ{5LKXE*j@ zKdJ|se=zi^aF8CY$*99&dJM_Ft_=ytM0*(K-?zb@#|Zw|==kUul)tGDf{HKtF3c~D zb&1O$hZ8u7)2Mla{kuo`->0A6u3vs$znq=ifaGm+0~~V>7jOyDTKp?y53Zq2Icisq zI*|YV2HAB=`MjxoqH!1dcc%7#uC|_RUaI{kTZCK5X#UC#>Ga|j?x1C+{su2&&W#bw!cg-zN+cEd1LNY#OK+R~3mntUUMv5RF^zwiLym`Q6C920-|2y&%VJH33I`7@|{m8HTIb|H^ zxN#t|=DPMD(KlY>sHZ0o%`uDi>K)}jufO+ju0S+rV6X5&RD9;){D8Kz{NER~QQEN# zI`?Rs4r!zI@y)+s{@ZNrAO7I`yEnCeXhmZ;`yI#nv)|D?l>LqtdMon#3ly>6#bwYb zt_zR$_dV=dQr?Xic@%LZqd2sp=BD`z?D;x&e*Cg=|5G1Zzj(ws$GUFE%O8M z3Do#+EgQXVw)qF%iF&qtf^6`PB*|O;^YQrscZBcZ0kU5!2yNSpe;fDi*kSzp0Q>(9 zb$gdOK5eW-pR{p0`@h@#AEb~U|GsYgn|;}eOkd^eqWSU9Me84x*&^jj+@twD(KmDY z{k1j+<$fV`q|{JxnlSx-YBh6Z|aw(>P!`9g`f=-Zs5$T6rsVqAfofM|YD zZLK{Rj#&HXdh1+w;<7c6NZ!;}yDuNl4G)tXI|b8FiW!)NIcVR)2Z?C!?G8kHZ+9X` zkG^%*w9h?oPf~N;1A2O?dw^Eq7R-~z0xZH3EW-+{!fJfnep)NM5t~u@TlehM5&8v% zA^T6meZS|=LYesO(Qnv^-Pnu$IEcfD;V7bg5aVPLDYQm$|0Ew|4kx0pG*6Pz`oPm< zwEy3kD6C#T7lps|XAk#ZjoRa}@_66A-{K$5FS$g&g7Q~Jct6xX2i%)C+@tyK4RVN0 zd?u`q(!w>|K=oq#|IfE(n;w0WzHYX4;!h~=uXwhHo(a(!`UL7{J`-*^Ce^k z^9v?RXS#IA&Vl|n!W;l;H3DFe?R{;vxa{P+oe^B$NTeN=AS~eZhoiu-PntA?_$d) z<$s;AE8#kAO|-|(aIPjOD^1TV6eBfV(9rV8+`KhpT?59Gj<1_e&J)aJ{M|?VbZ{VlGu7RHpKUnwK zu=na`L(Xr#Z~krg;Z1WMu09jqoc-CbZ|F1O$I-Q^!%q7BG|r&=bDs)t-PFE+?$hD$ z$dd4n$4kPY1H-~wqK(A>l_&L&E+y28AEi4hsA5=7zrvd#6sY{%u0o z(=;J`fAYAnJMkBx=kH3w|6^RB`y2M;SH{jc$JU$r70A}~3m1eh;R>SX-}_ui=%HUj z_dE7$FsRV;s_!dwn#(V)7q?LU78{t%zQU#@AE3|w<@ZnW{}nzH`q2lV7=thb!%%`z z7=x|T_1F8WAGCYpgkyJ&W9pY3nb;@1{hsnSdtGQMcr_gB|7u8$+!C4#{w|~k|6N$% zTqiKWf3nZ&=RB!@{XO4T6rO@?q5Afd;ix?+Yew2{ z^4%x+(7zMn{lCL!^_`Fyu^}{U`Zhnzw?oZU{g0daABd0F|Cp@*F~4|{eH^x zLu79q_Wz1+8&6h`9Hqz6wnhH?%72mklbs{vpN!@RES3L- z@;_IcHGWCyq|k~CTDDlDhg=kHvbJw=-*AF{61f9>Tl=i>v*&+jZQpQOTr6D}&XDI& zJ+*JRKwd%>y>5i(db@A9BHV+9v!3f^&)K*`B6{}i#^Ce=;-~c0$%Hzg9*_3_xaL1M z(2HBRgL`;@KC3^P|Ifyce2@A4KMp@A4nKT@>=;XA}AFcQ_A99247QPbqQ?s&CtClbnFa z2NC%m;+P~n1^Mp^Y>~&~`bXHR-?mNv?RB#EU-TXI3wnN27`mT`_WdXfy?>>?_*|iV ztDX$g{J#|4{IrC(@gg1OYBoL)6;X=r|a0KXxqlVMF%qIL>IEiVUV@!=7 zx;53|5S^s+GqYfJ>Jd!H3z?pz8%Ty{wpqV zTUw|Tm)c?ee`Nm;_e|+K#qCCV58Hp2`Tyqn?-kyU>_mPEJevPs!;a4He=yti&6MU+ zX}{zgNN!+*`_Ey-a1?PQkwPolj<`SQC~|-My1ygb-vaj+`SssL?(Zr0_oDlI-u-QI zf6-Fw{`z0+xce%d98TaQs>M|~_B1{2|8@RMrG+wICj&wqcD-pI~xTFCxis;rUu{kd*FEPJhFrpbS6l_P0o z%h?};*&o8u9-*CNO5b6W-^O4ZCZJ`X@~JFOqSt+(eBCzJk3J0z!pZsU4>EDp+-7kx zeU2IAEL5L0mO#!!)j;D0Mjncnhne04j$5UsGms2#YOW5l4Q@*PuamuKjhUXLn;b?_0K~qnL$<^()$JV z7ySgvpBJt-KV_=+0eSy)t@c6N`S|<-UHgKLQ`(nzwJ(>oFUPen$W8a%J!yU1 zK0PUX8fS1TdcU+6wWyoT4~_c8+BZC!hgOr&{-w2lUD`jgUO2%{Z_r;!vf0l`=K?DD z2%~R`U7}w>53ZqYjq&}y`~wC21KQ?J?Q+yMXU7`fN8>jB1sv1HrdIMVXn)ho`4z zKdk?b3dfN|#XreU(C*VS$l(M|;xx|S94_Dzt|0mjK>7QxhRELT5$^dypKy)55&ec+ zNWE;GEqM1KF@%kUnJ@UT~5;p|H5Y+=EWEEMRP)B_2 z)ljk3b-RYT54;~&y&v7)kL&D1bw?6o9N*1OC@(Av(fWa&fBmyCE{gj?VVFSnzG-fb zcOm*lS~OQ>lDH|D7X9|V^1bV|(7x|AeVo@qrs_35t=H5~uc@D2Q$M{H8dts+n%2A) zj%|7^q}II_nis#uxBgmaaa?QqwNSJ0wNShKwNQsR>XASLk|=eal>0Ixf<6kfFbDHc zBh6aWA&&YzW#0cX{`WHe_p-3SF^jMS%di5guo`R8wxdk{zfAwXjQ_oi|GkX=tBn7v zEaaA!dH>7!-^=*l%lO~R_}|O;-^=*l%lO~L-7O32rLz&6QHGX*@;LOhu$`Wn?B1Y~ zz7x5=?hU#e(>eOJ{I^SD&iCj$bk*+1ZpUT6Z!eFfzt7iRar@EmlKX=EeR6CKy>`Ci z=PS_vqw5BunMd3X#UVzdemp{z()FJ{M+^WAOHTfHY}7m zruY4_(A(+#rFVZ)z3{L2@2{~Z%0gun_m$VePO|6ouZ2G3dc@g^b;!?aBa_<2=vy%r z>deY#)zdrdx8t}faxb|b)%585JqPKB5yMeD`hHKGp6SweK$4z9G$xSUru`&4_i6ve z@_(als{R2Ue}8wW|FBP6qco!b;RH_NG|u20+OBK=(Q#M%k4}0QvP-r9+V{u(n*z4( z2>ly*OS6rmZymN?H2<$j{W$)k{@)0D_(-$5*8IOC{lX=B{Jy>kuF%Wh@lKr5H#x44 zLe`&mo`3cX<)g>%*U(a{KSA~)+J`W|E=2ub!M9uar1ydzts9Hh-gV(|{uj$X+dTjM zzcucCTKTUv{$mX2mbC8R9v-02xAn&{0RQ3qql@0ZBi=ux(2Qu0@s>T_Kg8JM#nKss zAsB{gwt1B?pb~m~rLv*?j-ro2Qh8|*m)M}JV4S!Kn1m^qhEmMHEIjr<@lnqao`*%q zzF@4DtO)Gtr|l0&kLKgAAk)*?2b1lMLtl;D_x@b@Vk@9izvc1xMnZqX?>*ls2qVZ20b6jer`@hBgCtJ|E(Ek7CVs(F&&EFjP0Q)_h z|JPstMg5m?jvP+nG|u4v)gGK9d&%B!DWBhT|9<&oh^dn=h_Bl8iExR$g6ehZUGf^D zwG?$o+|`G;>D{`n4?&*r+XCD`FK*!u?%@Gy7P5~Qvq9OaapT(c3AX1xHmGsy(zZQ|Mwv5);Hvco%#%Xi_?{(PzaPlZ0~yw4bbVzf+VBlqQ7pl3Gt zFNV;EA-9n40bMH}&b2QQ7n|!lh2$7iuX`$tBPXEhCD%RCb))QDIch* zl0SK>BjfGjFN;4ee~9)P*!sakKkn=9)g)==*WWi=PfMSMwo{%zqCHkJXFsy<+~e#4 z&wroizr+3U>>uy{=DDYZ^UvRLsqs&RQt8aVEX+a6OY$#|^XM66wL^FTeGziPS!KA3 z>})cxSX}I7>&M9zsP6kzSVgWzm3-C-$K_$I@Om^{_3VY~Z|g^rzh8gPj^@v*KcsI> zHXxbSZ}*?gD8qJCVkdTEFWU4g|7ZIFrt&-|^v zVGMm7CSVeJUwta{)_8w5co&7Ip%gRFGSNFWxFF1;XGVB`=yT}vkn3g}-dBgB6NAJp zz#=R`%=IoKS6~%tc4?orZ*_;XPgk{1_w^CZYM+oqV-x>DJO9CKwlta-v#-tnYeDpV z-N)bmnW}w8{`)^;wVOk=&xrQ_U+p|=u^t;yz0UqO3w>9P9@h_u)+UzGvu}GRwaX1% z56AH9$*uFX-HW|H-oupkeg_9gdW-XKFdxEyD#CE}qkAyk`A0kdbo~P|i^!K>vq%0= zhdAoxC4mO*coLP;*oocPi=T|tW=<{$Z}V**8CVz|um9Aie*FCx^}r+lPGgn(qaSdL z%pm6g)1L~-_P-1b1eTYQJle`80U z2|uYdF5C3k@Xqa$aP;c1P)#rYXEyp*hKDnbIfo0lge&MlQ^B*^;J>ss)P9P^?3Z|hHGcXHt5Pjco9@)eHzaR?p(JqR@^~Qhn10vse zbbYV-s0Y;^=H=7qcw*eSWn-GT)TJk;zJ)~ z=WXxb*b(8DyshxtW|X1&z=%+_$G({Kxb=W_sHE>ia+C3Izvt)o9fkFR_t)FHqA?|0hHHS)h){x`@!&N(J^;F)lNyo46@z!kCw>858w z$87Zit_j~jcDjCnI;3-|XP_RbnfXkpjpkS&I#>M`X&=xxd&9Z5>HpVz?*4ZRcW@66 z(C0heMYJXKKlCX&);JHD|NbV2OGQ|QxBipT5oZ)$(}sq?u^^8kjwD*zoukP*Ec&DUm-F9eGav4JYfK;ud@Gd+iQ*fX#L+H`Vjol`aeGQ{QL^_$1uN_AX@(y z`Ts`ID}KuE*Z&wpABPE;gejPY{QAFL{QrmeclYU+tDh5SP-c_ImH)TRvv%AJ%)%VZ z!vZWq+a~WHI^JV1zN=4nS>KM#BAV;ns19kuvDwN$9{Zo@tx?=y-;a^bGOWNVv`qE> zOtg;=Ju_4LfwlDY$Q|PULl-)+<5g>zv^5+3wi(r1`18r_sM?@h6)0Dz6yAx3i^>)1 zkwEl9Z$mV{VYlC-JqY)cl{NbJo%W-qA4UvE5l0d!v?7BXPT(YZzNro;qxSa<8TkUw zkr!|YSI~oNxPh9Nm4DQ2a-FEBC-A8JXX~E~y^gzuJ9zA$tmU5+ZkzA^FLwWzEC1*m zs{G@z{r{)R|2pL#t?2n#G+t+{|L^#aI(=_in&qFdmtPyZ{)K4$b8+bRx86$>Bbxg& zh#V5>-?2v~S%OiB<{ytCAI(1=N1uR6h@N3)sb@xZ=?`SdX#VkD=d0Rcy|g{7`#5e2 zrlAxuYxC>(Sr2~5dT=s1*8F_&vrs1--(}9hRr~Y_&%**NLW{Wc?We*LdgciG-Prds z`U>Q#*zfG@t|s<7Su>dZJ(``$4vv%c)7jr-gK(0}+lL?Ze^&YbYOF=Mv6%H_n`>%E z$5j3Wbke(!y>FcVw)F=~+27JRCcWregU!-Uk2L;HZyjphZAVK?fp}}JCH#q zqHnuIdpzdw`1=d}aRSV|D$03FDn^St^`9hfC^ z>cqx0|ARD-ZQ~?!X7|1p6Ob?|;;u z==aZY{5&ke5-daSubv7k$W@X4Ezfj~b-wic`u`{UhPCt-agmQ{Jw5Y*v0-eaZ${32 z$*OO=_yIb}eEY+`%zt;aOS{fa&3|8qOb+cA7WhvYwxbd|u^W4_A8qXHcI|CPk@mN* z_IHH#m&_tJng3lI+k|6m@|0tn*^6oO-&)X$*m&ay(m9M6j-pyz^j*6+J^JQQ9g_4E zlKa^Iwd{X>fO_)d^+T)QGsvO*Yu>SMgm8j>5*5GF4u6HuhkgdRlQ&H z+KXrob|+agTK`90>&VCBAM@Gxv)zAvqpjZINBxU&{yzcb&q>1^z$x^$`^vvGAFnT= z=l7q|w&eF8>d*gxW6FB!UG2YqX_}0_5!Fgoy!}Fm_FbPQt(ZA|rDRpY_Z7_1n?aw2 z?6dY>;bVv+dS??Oyu)O&$U2vo%nxwPJVbrY1!U@o`7`7aWM?Y7bCt^t%01b^_m)}1 z-|x2-ScTQ-QI^+|>#-4=QHC1rRPDl#_0P3k31zyWN&Uc&*r={(x~zPYsSlLz+sgNS z<(q63Zd1qP=O;NfqkhSMf0)c7`u=eA{p{_|Q;D6}jePmNYR&=uQTg3ZKZxW|c^~K+ z$oi4`FE4vO$j{I8`_bq>(Q$~@5tqNhcloA!FWh4csFlp1;_L4FSFQ1-pTJ3!mroC; z$ul^IUSk8Rk)F?l7@aSf@n<_wd)xP?22#&)9pzwgmAUEU2mp!eD2 z{o5c-Jo)-U6jv{PulY5g)B-p(_A_g>eI@fnsZ67 zKKr8mzh4X`^!R<(hEen}NKSTbi1sL`$2f7RL#1H?ISDOQrC|y=4e3N_=yT+%zb<8 zKUim-xVY+(u79j`;`F$@*2!o2=iPTCkGp>NE79%xufG^-aKKiXex=G~6 z|I6JQY3;;r?8SZ@MB5hkzrg)R2A$}#UqE)^i{?4LsQz{TH@N?fOOefs-T#^Hzh~P@ z*6dKe_9 zs%M3*>d8m-?+yR&MYgSPxI^AU+jaFXI`p|RJJjiil&x)^HF9bFui5Och3v1T>HzZJ z?*G{Pv~RpgC%^vT=Y2z;|K=WiUHd}z{_Mptfb1rV$(}FRqd*-pD2mf>7(%ve`9v5- zmLR>sI4zl4$qpdLA@7G+?z%3!uJf+*qU%)ePw?9$#LVx$<@{6V(@=_P^SZ0F1vBW; zx7h27%rmCXLG(?Y2JK1ot@e6yp11{AgeBPekuNQ(50?pNUooaj=EuK{=d7Zy##*dL zO}qJjsMCLl>pRr<*S707G)&aykL?%A_0KjsZZpcT9hKOL-Do>y{0SWw*}rG?Y3}OV zAbUXn4~2c&f zFnK>z=>2|SI8Ao{oL_|Od7r;zto9T%uP`*8g#Tuh1i(NS!f)9{M#zdlyD~807u`8_YEjmr}p?lD81;D|v^!hcrFE zFOYZdfpDMAo`-vrLw+49S)+{9M&$%?)Th1w%4>dqtOWnp`%i{`{x<-{7=$4hh7z%yzLh$z;@9lvZ>l3c`DBc3TvS&g+=kB!)jGHgfN2IYT?`%Grk6`jghv<5MIO8Gyo{HOm-{kKrt?0Gzze-(Xa zq5R9fyYQs3Ch3*`=*h5?+>O21kApaj7$V>1R`h<(nq~5+xHyu?{`ASPquKYuKIwaB z`o2Bk)1jLms?~2jU-yo@ppQ<^p&~pHD)|^vhl;`s;S)HC(`aG;rPD>>3_Ww#-i0@P zBJ>*G4uPckUgm0QxvX|H&Eq&b<&A<`No^@Ei|xclWg01$0blh zuU*dmMI7~b{Czw69sjw92k7%EI}HO+j6rC#mtFgO^Zb!P=Q{HXkVOuS$N$a#d%|h7 zpcSd>#bJnavReznFf!#lE!~b0Um~38%ZA4o`Y7b+S#ajvxYN9cJay44Kb8E@CeNVm;XGxb@qvH zu<5^qA8+_C;lSDF!;iZEbJ)M;KZhT#{LjYvO2eDWUkrQKy%>Hld0N=B?!SibZ~Cuc z>pu12A@$)A_xs4@r^4Iqd^xrH z@P04Lhhdj-8UF$Hkpf7e>B<4t3G%m#sf6m}YGR zoAsyW(+n&Pl|xIdKb#hJEd8(H@AaE@j-C{f{+Gf(v`-GZ2Tuv#n>;D(+BV5}>Xfke zV*il++0fAH_a8PD*;}(Xyt!#W*tdK@_;Fu((I3zFEr%1}o(OO0C!M5sf4MlECZo0e zXUKE7fJ?Z7rk92o!x|D&wKocm75u6oUGRQE^S~Pgz2D<6@O$mM#?T6e^RXH~-4l(y z3<~x3_)K0L5*qpr3P*05qu{vMsea)n$N41+J`>(q>09mpA7%F+BV|@6`v3IOILjNZN8?;r1cKF_JA&hwn- zdCvKq&-rPdWBt8R#-ACVzg#>268_z5{JS@_>!X#PT4}7G|K7j=ZlmOHw|if>>$vN! z!J!*H2lXN7C+O`qHw*L0fTho|UC_t!Mex=KG(2i6vzde*%OYeDl1gToN#q*wC6$>GS3vnDP%3K=%E zN83Se=Tl*<Llt8`CIG{ifC)$|L^4gtLLN-8vjVo zI4`U2(5W7gs4IWp_{TNktiM_w>Q0o0we;jXX?j(=5`8058EF>Qf@b|jHD5Bm;)C|7 z^ILkmwtq72As+V*-$t*+4*X~JgXSg5k>%S$K#FK=(WPi4@Z4L|iv8NA|M`g|Fxb zr60$@ug%9NPvH#CA@Pi~I4|yx)gp-&d_q_$~MI^{2v9n_h#du`f)iJvDInm>o zl-LJ7nLZW8JL;Xtzsv?8u}J@;YfSUo49r6P>-xX-^HPi~jG^u5nTS z_gQh?8l!D}OlX}wCZwiYTmE+;taa`>tj9)dLKU{5>#Y9&%liLs>i@s0|G!WFKME+K z&3yg#MPtHIq|tFp|39+mM1%hP#+|M&Ey=j10omfXwcGVGV?wQK?7%MU!9Eerju^;cv4+Bb?6sO|Ca}Q5bIgJdGCLNJhqviMYdh#|KH^Q-{b$^;{RWiH<6V;J2Tq; zKWd!`@$`Sf9v9@+y5Pn`m_vV)EcJi<-aL56^RW<%G4Q1^VJW!`E3p0Fe_ytWT!a2E z>k}YjIkt|B>x!%=H)6fKUi}C273o!o>)%Wxx1kn05N&8i_%3>`k!?B5N2c#Xkqs#z zU&q$4Eqxuve+lpZMtL|yZjtV-eA_oi_%0on^8Xi=v-BiVNMqoc^3X{p#GfOJIF6Hu zYxvg*KShsg5;nP?GxT#vjpqNNWrFsHDST&Paopi0@(Qw>jin~9A)~*chyNeP{oQap zfVg*1k*zI_1JuUQFkAckR@Y!3lY7P4Ep9v>|99Je?&3ZkV$c@vrCS=+f3}caob7x@u%b4=Kezt%VsR|RG9(_D z3q#gj_Y6kcYlXfBYtb}Q`J`NUyl!yo8}3hDU+1^=*obOl2TJWvPy7C6YKs+Cg<9-D z;Wy8OUF06re9^uJ&**31Kc@kO4~KIb05AtZ1FNu*HtWf)Cz(SL$8i$X z4c5*l&)^&iZ581H8S4g@$SXLm4D2Vbq5tnfxIxD84+G?Yw&S~G?Z@K&fEA%MA7h;7 zGr{-;;px*u!#46Z9%9f|-_Qg8|FNOrDSA$yaZlNY!Vr1|ijE78^JK4ffZ@XWwONcL z`~F}pdvYwsV$ruCYxPC@m!7WQ^j-Ux)#qRRui6})n=uBU=d5-Cl*-Y6jped-fz{dx`2P7f ztpBG!zwxF%etr2#H1nlf(25k+h-WRDLaB5Ej>cI1pH*C+pqkQ}iLIK-|0ZN&iTl>m#+-8fbnSiBTAf@#xo&HIeLVR_>Am?B7&!8cP22 zRr&9x{@ZxYm$Y$w);OR~Yq$Hxhr%pjl_-3l%_kp^|DR7^h{YKAgR%d{|1YJdmEp_C z;qvYZauu>el`&6SuYjKG(>DIB`T%_$if844>+1iAZRO{reU9+;*oaN2!Zy^RfxT*E zubSAzB!9HIkL|h123_T!p5WiJeeEx6U+1@`+1d^=+TCb}J4YM;&-Qk+w>@NTHoL3s zJwJlqJ&YYjVwinqU1t~eU?1uiu>bRn3!oD@7(A&0o9YF>VP;(y2Iv-4g5d$sa^vv)(rF&dp@dWtrD@eJTL?jpNL9btm? z2k1G^p@*G+NFTJ#y&o2Lw`;UV8?NpkETK*?gsedQ25N>%7;4-s>IjnV#9}-Mp&Z z0*}`*YNe;pj23L^wpIaO_)YC`u`j3Q@BUC0+Y1Vxl1JQ6!)o?-1N*yK{vw-SlfN_C z_w)Mc$jQ!`ifNdES*XMu%thCtcl#gpCFIaMOg}>TBaussVD&xwOZBHxX$YNd@N3QYPT2%kyL0RGFe<)i= zUyqF#V23u5vH!n{?6-c!HZtb@t=NKS|NFGdm&$jUun4(!4{)NJv*e*2*80R0dW zID!Fn!vXzDiRs!j9jA~+C+hEL=bPYp(3599TXmWu{WxNssnv1Iu}5{CJ;F}n6wYAa zXAjEGkr!|Y{oi;{c7^Qw>>tYd$!oZQ0o+FV>N{ocZU49{EGsN?=bf_q^xVpaWjz<) zDSJq-{x|RB)rVyTVfhUY%X;5XXx3B56e1{sC!s8QaqzD7UNMbtWF#g>B({KO}vxoQ<0kF-h{PGcds+wDVruN zef^!X8RRTv_r6nBNzOq=cu)H~Wx4Km%H}$rkJ7#{%7EV8@08`yFk1ShwMkl&(%wwA zEOHH`u+V=NV=0zl1y*4V)}rg8@&`RPl|OyrMIHqd(e{G$yU(L#@06v{A#1 zdK~}vplqFZ3ZK`{N^U~^*#~8H!mH@X2M@}c?mQ^lMz2LmJ-n5jZ#n;k)?Om z4GPD_aT2F+2Ip`A1HaXNNA~GIyF&JVRo#NzqTak!A8s5IbHj1^o-u@E%~$mM{Jp+( zdUm-w4(`%(^OPHWmHYIEC@ND5$a{{xi_|x^yT)gH=hph{|95MKI39uu497@}!dQ&o z267@MW8mNP(1M6jjTjHeGWMnN%_6$miN{!-(OUB$#{?G`&aqeef9rc zc^LS1x&2m*=a2r=Y~v68FOFMTNG`@wEJNSto%?UrJpFn(n{OWyVWoN8zftd|$GP8)~rw zyU?|P|Gn8Y$lPi5i}UOjdsZNej@#JYc5xp)$j>56>))}P(Z+TnG5RU}Iln90Bd&co zfclq>|DXO;I7Ck_V&mqir_hg}H2!ce+x8M0C;XpYzE^gr?q1oCcRwgQxZqydkCrH_ zmp>@`;ow_k`;`ShsQd3_`&Qm6`~GWp%JxpbQ}(^S>#p@b%D%hgcV)YmUoUHKA0682 zMu+rU&xWHfnOjr#@z8PS*^rFaLK<&2%43HImByO=RKG)F@Lz#Vu>v&%!nk+V;Sdi_$3Wgjue`GX;O;v=Ey^aypEVWC-HQA(dt>xN;*Aq0|C6Gss>eRs-ghy9PT>Y4woZ2QjtR`w5X{ExC7eYeWu`g14!{*U(_lA{$!b8i4v3PV0ZVtWSY#d7zKsj((NV)(vz{Kd#{h25=j9aUWe9^wVtC|Mawe z8}#~5zD)lUinI08ysn=HN0COyBK2i7%Yw8#j|J{N(84rm(|*cVQ3qA@;G9%BQ?FaU6G@*8hK=pN|}0zL(FR->d(B zH$NY3^Z4I5`U?LW9rO&c&g;BW5fZOeghS$qYvUaulc?ACUMD;ik9TStK&k%mRdr{2 zD_YQuv~$wYmnUTv_|9Bm;ZMyVpD>(YJ^rM6A zk9$L8(RrTzZyXwKh-Uz|aTlfW|8+ydeR?vs4HzHrkUpr^`ya<{uw75a{}1&(wDm@N z_>|v=paOC4_?mCZE59obBOQ;zSd7O+OvY4{=4H6Y(mIqAycg;3JZPUpGUK=>@4b4? zU3K0wy)eal9pOED$N32rVVXE*U=}Jd2XiqW3sG9v!8>R;;W=OSTn~D#o^NxV=e$?= zQ1y=yY$K8j*v40+4XyOl%lbd8OT1V-OR)^qUFBf~xe9C0^+5Z_OP)V+=*9cTUu*y1 zGaNmw{o{o8kIU)?WY%#f+4nE}{}1cS|2_Zz8SDIua~;-WBQ~K5g^#O8k*dI9;BWCu zdBgFWtF?cz=cV?q*R1Py&{z%Odyv_y?(vp(8PqiR|8KmH<$M9@ihH-kF_a1A<#6Y2 z!#*59{VV$4m5+z$No8ac67(ZTZB;h!S4QtvR+C9#u@4~?C9?ERGKbiQ(8H$3b$N@9 zkE7tY=(snd4nQ_s<$vGQzCb1)u>a(f_J!s8Tm9!0&fpv_;1aH&A5Yf*dBBgo!~Wl7 z|Jm=t0`{M5o5220VgJeWZ1$hbIL?x#^?zP_|N1}I#91$_4maq@>+FBuQ(=H!(59Ml z-0HZ6Y~~-|79Qt4mgW~2m!I|i?+Uw*jJBDc1-^?3#@{&}^lfp_i&K0jFZfQ-)}np* zg!bWM+5*r)&zu&|E%_5&W%4(Aki#GE-}aO^hoAz(k&sVElA|yd4GTP9?I%sro}BLa z%Kt4_^=}~+^Z)t5VZ8GuVlt+pelY)0ewap2u9P=?zcc8wka|_#5cYWepGskKFcCwU>^?P5E2;phWbkoN1T^L3TbpAha!%n>s9XuJx_Z-cryQwUL5TGX5JnDN5-`RJKpeq zk)?O8)c*eqzW>kE{kyaSeDed3+OwuVs(+sp|0$e7;Wwkh1@aOGzT{c{*7#6*?ECB| zui-}g4RH)~-23D<{qg*Tn$LMI`rA|9e{2Kd-i79I(!X8*_ip{fNa~Mn)-T<1UjO%5 z{lq8CZx_d1+{Z%FaKHjiWpDb(0u>61}(TzEzQw?6G&a;mVz@-bl=IRo{>^rw=Q zh+`O<#FHFs9Kp}jceFpXj+QRR&1B06{om5uD9ueszAgNsaJ14>$Hu&G{hvAFtg*I5 z?bofXLZ6R6yZ+D1|A+pcd-A;dTPU8zSc+xHN>@huR?u^A`L?#&2Y|i?MaKolagJZ_ zZtc3l)*-RMJ`QBvvtz?G2Z64TI|3s>_PQM zt?f@9z#$}X1WBaOi5v=Fd*qXp=D)mQu1hF?wEx}LNC;OVmgp*|dzmEy0 z$TK*H3%GA7)_>b-@ZsFyn~PVyZH%THGio~i$Dru>hlo$~*7`Jer4AzKlBSeM2d)OQLzVaI_+YqW>JnNu0tNoWliNLf3Zb-{~I7xL08>J?;%uP`@d?|69_J zqe!2S{$tXHEIN_yR*s9OAJ=dL*$HqV_(tJ?>d6crDgpBQ{~c zGpi!EA#vIIm}Jdw0?`_G%F`S^V0EqvyF$2U-8Zewj*eBS!M(er?xdxw9| z7tSx?r!C^AE#QCiY1{F{|9qBThwM;(-bMY_e6RuW-Ns$qNBvp--+p^Yk8J`?81x-w zK2n$68(I+8>Kh^~?Y=6=;mB?^o}V0r%>H-R|6gq^zvJf^x@){3 z{b*hy?w4HyajpG{{xcaGh3|98KC z|AV>WtUn?B(l(!-)GiR$|6fQi@GqY9|HpfssSj9q9P^W5e-_i@9J6I)%{ROsb3lqS z)fe~~`3c@Pa`d<+=^)Q`h4WTn4XVqv1-)g>5BlTv@7B?apO_ielTF*@$D6*{d%j&f zUjMHDi{_gAobSq3ZFJryRAC!xu>-rX2m3Ja3H8-a7|%fO`&-XGBhSBTj!2n2|FpQt zIL0USf@{#@x5?U~+m?p4qDfoDw?92d#9 zDe8aI`FGAqlO2xVyZ+A>;c?7g-2ZgIUIR64<}dtOyQjDZv<;N@|I;?m?>NqfxJE{Q z=SDo%UJ&9(T?V;Y}~If9yju z*|LDGAXDr{h2Mr_Bob$>g+Ru&>FU|cvGnnH@BHA2jwfR(V*mRzat3NXr<_!7&Z1Xh z4(6hvk^M!JzWgNOoc}KaQu}@Ta{buOOHa^mJzKvunMLOaVbZczJnOI?8xh+u;{Jb|=(+vu?_TQ<(zl_= zt`_dGzqi<5vQ}7PgE0)`F4VKBd&qq#?f-YybG@t$-0>l_<~`Rs&zWq=#P$#C|KCwJ z_uVAPX5USV@1)gtks`Mkqp|h0@8g8}^_P5C-_TAep4w*LioU5X<2kx7O22U)ajZx0 zZeu&7yRcXK-;(|XzP}~DzvaF^^3nWL?L@cCMMrvr?_PW9Efi99JR?m>BqtTyhY{+2bt>E{slpls64l{{~pkK;>db)3>y)l9aW zC|#R<451NmEz(rp7-h64P>DIHe%@UApO~jkpO1lW&Ik+1#Zmt#e}G(unychJE`#9ln3~(_!ys z&tv86aA2@?9{hG3CvgftR#u%M&*1_t;R+JI!+!D_ZlGcJi`r+rgPSjgWS?h_mh0ZZ z*%!4>ncu(2{C*ro8Xa%E=oy;d|C;&z-doo=>jR)?ru6~PJIVS0BdiZF%=!RBUkqE+ zRktF}pB-?$g1-8`w%Oq>z5lb;E%>y)Yx+YB{Jt^_+Qp8b=7;5_{g%?wR{g>9Fhp1d zh9mp*Y~#=56?*QDz3nlIJ{HBx_O?gOtKzgHqdpF^hSSr?F;kGPjm5xvLM4bXE>T>%UIwipdl ztUqMDL23P=7UfdL_=CK5A~JPR`-ZaMasS3r=P$zw)LJX1=EvosOWD_rp6SX1;$~;h(5sCvDEa?vc77)N%eKb7!xQ4Y z?fX3EocjId1dx|dcU`*j(zR3j!ROS6zsk0-)g}LbuXJu^f8BSZ-<$leH2$9+?Q&`U zT?^k_JpH(a8yLWC+{Jx#vE|(>mGQ`-7kNBh=d10a^y8>>#Bn&TNEBB~^BdB?S^6fbZ>9CM|F@mR%nl~ zeK>%`6!V$MI(6!L`8h#9f+U*c)g=CSeT4pBSfl*I*6*`cPn@T><#p*nEKgJZSMzi2 z0$&&!I_WtKxaWbOZ;E~#CvggAa1Ix630E-iv$+1_(9pL<|A6EE4_il;yb*uHZKUt$ zZzAvGAqMT1es#WB|9Fa?TjblM5207!i9h&~Z&mmc|9AGg{_j@)?@s>jcK&aMe<9A{ z{x=e%@ZR>nv5vbA^8Z`-zjge7{(djtKfj#+Kac59R+p{h=_Go*c&irI+Tv7=x91!@a%k9?ST@!lyZB24*3hsR)&1 z#yw=+!yNZM$MIbB80()yufF{!_vaoPUgd+LNt{WUB@Jcu?TjuA~gfOFF5L=MI95$zhjU7SSMX8He^Jcb-fcJ^&~O#UmPZ9Dt-vi!e5 zUL`x0%bR4@aVHtq|2ZX|GdPC}cszdY5xiTR3H0~-ra1|K{kqgjeGsH&13&?bU9ncPIW9`+itOc=tSam;|#=;z!4-- z|Elo^D~R4>60#}#-o{%EsrV1xI6!ux;Q z`@iV@Yu{=14YZ)uHx&DdhWpP*jKWxaC;GYML`+84P4!=Syhk~b+p7Lc&(jNe<4lwd zZSrP2jyf-`{@I~SiF*TP(W$J7Yfn!V&os=yEY!c@{rfH|>B)ogFy_$b;>q|U-+k=A zit8xN7rtdRyZf{?zv+b^8UsfzL-ogvfg@|bVm#7!o(rq!Yp@pUupS!``{qjH56q*3y5;&~@7Qf%C=>n8TkVd(FwuKQMmay72>OtF!)myY=Uh z-f8{*z2@)VF@N7Von(Xl?nX3C(SNTmy?KKEdwMIG66ZtrIOzNG1BRdi{qo6hawO7M z$A(elSd7O+WN*3#Cew4%rEQ#X2J~qtzAkP0)$^{|>l$yqs{P^c*s$dl-`^th?;FR4 z8P1!9I(pM;_vJsOd0fBq?4>)dYtX#XeR>WJFSvg+z2rQU=H+`Xt^SijrFiCGF6Lt) z7Go)vp=*-VdOcbH-}_FZLz$mJ_F3akkoJyOh-VenU@fx3 zGFV5?4fWk)J$)mJ^aAqe#U^11@3)HFhWfZy1i1rs^w{^D#4g8s&`M9?alc>5zaHlM z$3DLg{8FD7c?i|t)h715>wQ9-#ILn`3aj}rd;2eZ6?zKkmFf^=Cvqqvt6m@17(5=s z)Hh;Z;7R(a_DLg~1c++`w!9$! zkT--4M7}&W+$QhhJ|1GwUimw=AH?*H(5C3P?>EndA!Prz$A${BwCCz~wE2Hdd&2YO zVWhB882Fhr{(nCv^acE#{Q;i0PDk6AFxGG5F%gq771K~%T^{28f-~qD?TgtFWnmV* z5_9lm{Ma37f8ZIA#hcn(*_y^Y8I z_l5P2HzMx+ld98CK(=J~TdzGT_v0FzoBUpdxCZq$vKBR;<@bLngdOx<*n@pIfJ1n) z|4+B;pt(`{(drqbkZ{fsB#}ZIoyehxF3-FhrFr&b@5M)TtAhGfk#F{7f26nAe{?L8 z|Byu|V*kT&@tnjdoWbM%hjaAYW;T3-HVyhE6!)?_$a_w`xFRg}KlGE=5c?l)kOL_7 zKZrlBD_q^|f5tC9-Z!@SS?NT>W%d7?Yy^A!C-ZA(4+?k1a~}^;`0_Jh&<~WQ7=j9P zsq1&6=c4-mS#|Pz>gFiySO4G3{;T7+tLq<)b$$7;gUmRNV{|&5w{?j&fO&pf&6d++ zzr%3xjzpGT^8f$sQQJb#TjJ&q#Cm_p4{#iP_EF9qi}9F7577W5}4tK#zMv#Wf-t4$8-SeYf;d`_}8;FE+VG6}F)kJFp9dwr9c~ zvSA$m7fndce&kcOOyOhlE#qAKRnFOm12}{Pjv$E?y7cpR^R;{U+_~5I*BkiP`qm3% zk+0pxzimIv|3w-d$TaeQck_Rdrl-Yo+xOK;=8$c14vO)3HQ(BO9jBi}@io4+u>1zT zHe$X{+~gO<`X9ai`iH_P=bb?vy-9mP>_b22_ySt#aUX-n^ZXlDO22rTHam}O7Jthf z>3`rnvc|ZxTJQ0a|6V~quHgm-@V@mQ+11$Rls_S_p_Ko3O8;KzKPa8j`zQPFy2fqs z+{Jx7M0SJpUp3}opY$)1{&^n?Ptk{afaf&=LflZa)o5>c(tz=3*o9sVRF%2^?3zaB* zT^l>uwNw7*i}xT`CvWEE(GBd&X7+__yDtCZsJx!OCI4TP$C2%p|1N(;K;ovk|35De+dI~UrOsQ1dSP`~K~KIW zZKz>iYh(M7GN=_TtEE-=D(CbEV^F^MRcqN)gf))WVjb4waO0rxlLzLB92*pVI#fMG z8}tC*zF(QN(eIm3h3fM1P{1~N-1DoJ+<{%_S6|sf_I*Jehdh8oNZ`Hw3r8F$k;1Os zqr$+C&Hwm~vgu#s6=D7Vu8oN-qUNL8b-r63_Fo+teux9ttu;7vNI3YwI*hl5ghTS< zKgpN7Cyfo?Jv}PynJ_kdPhESjwE@3BX?WQ8>_~gD4G-Vx`*7Gf;ltq{r+hf7|#{LxTz^`oKo)R_ zqqF7KX)6!kx4y^Ts~-$Mz`o1}!xg{v<9n|te{gtr2-h4Z%Dx$Xx?lfY_cz1Oc7HP* z+5e4D-?uq5Zha#(9R5Z~zV-FcRQ7e>?AJo`D_;+(?OzM62fr5D`@S05%D$@q@GIfy z=&$k(z7jfcL!1M+jk~yyhZyuj_6tK$fri<}A7}??LQ=ax^JVpl6UHAPHCns7bKdL! zk90f=V^Po_-^GS@qi2$~2yF(vcT4|7=N9p8wBJY@VR~Z;|#tq_EU~mSF`}q53n{a3SBPUHHX%K8aHt|P>f~AV zl5O-_>_E>w@uQa?o=2QRTU^br@LT;>_Wpq}%k;!*HuQ_%3^l+0W~lw~H??6&%Svf- ztvI$fuHg~)`aa;?L-=#+|8Cu`O%Ms^zIl@mLMD+y8t*IrwX^-%^?%j*-dq3c6h{t4 z97o?L)knT;PQWMBM}DE+_OZ?3r0@ZKR|6l_Cq_SmbGU%?a?j#(-wc=NSJ00*t|_hs zevO`!{F4f+7$p5z7OozsiN!OdZh@Y}eH`=~#^In-q~hllj!z0JO%&0)}f`39*G z-wLg0aZWSdyZ?B){aay(b26)|^gmaH3VO|Fm4)T(!}Y3=L+`yR^}{NCz*V7mx+)BJ z&Pa@Rtx@FIsAD1~qv5r0g~sLI3QbGC6_VSlLi5(D(1KQ^(DqVQXva~c(XpXQ9rDrj zJISu)RiS%IRp^=E`mB&0R#vJ-w zq_#@`Zt2f_OJC5p!hB&1u^3CS3@fk-YfxHmNxM|6r>u3n4%PaK3T@_c(i>Wg|M5PW z@MQeiv+C(jv(qTG8MzUZWGCt_%4_%hhMeOfS~s)7(e{!p8~p#D%foTuCsFvt;BbmOgLAlm_ddT% zj<2BFc;sv34a7M-17ywT%1i4E+@{~feLTdVANg)E1QqBvzhF4o_gU*^lA~hS$E(8F z7^b`!Pfo;SOhxrq2Zw3o49r3$20mI9=8$udzV#PjKDiJ}u?*Sko?oA}L+H8J`G&8m zL)u5M=9lG<#$&!B|0xGzUvb}OtHLVh^?&+X+8@iqn-}GCdjf2!Q!k>giNF2aw?p>* zi*Y>UdJI&5D{Lg2lsQS?&L(;lO6$8ERKHXIjO)8>6P8$FO=ofk>KAz5=lo>;iG17?0<%e=q63{~-IQ@wwO@|I7aQ zJ+6QA=5l$sj34&0=S8o7S{teVokAVGX|VeqYW#rXb7-Zf?y&PCq{Hu}`Eg5}hvZw* z*DgFSeT~k;asRo1OSpo5T*D0vplcrgA8|}=4o~KPsAI>nw#_qo@_crBKAvR;S$gMo z&&=<)#d8<;@er|YkeR6+|A6$rD*ew|=Z8K7#pUcj@~=q$>&6HOONehcITH167!yE_ zMcroUa&B^pc^r->qE%ceG*6e#(b9kZ-Tr^yYZA@V<;&UX$Yg4Y`uS--`n%=VWdEOv zI6tWHbz=kQPsZE7rGCC$nQ-utk6zfSey)DswnY7Zx%xku<_C1#Q~yVn-r1*K{xN>> zhm8vrS0(0PF6Lvve7=QbpZu_x?6;R+;XjNKpfAG;#IZa*{EC8#^eV?i*D4^7Uh$`& zHpWKyTCBr*WG}l1Y@|1M$BihhZ`JL&&`a$<=EbH)=u>k}6}F)kJFp9Tun%3&^4q6a ze_+BRKfd?kBfq_Hnje3H-~QU*(7u5mkMzs@_T~Kc+14MJ&X1ooIP{ql+4lwZ{v+&t zulXCE^_IpE4v72BnffKI8E}YR`)%thSSR2JJ&A!|nv-hoU}BQ@o&Ae&oPH9i&FVvFLGv-~|9tC4K5El;ej*<=t|?dQ|L^AiX7o?{&ncY2Ib6Uc zTtPp&^7_AxukYde=g3~Zf1WHjE|P8M`M;O>zhue(-^;gkoh;eux@mssHSyfQ0B$4B z4andwJ*OYMXV{0refl&MU*Z4riA(-J|Mj7;1V4DtkCpSNrw<`3P)BcCB3;Y*Q9m0L zh6`&QAzh=Tlm8so`Do^U79?qj@oCSo$CA}cK8zNgW1>Yma6 zpFy96qT>Sc;_gMIu!QF@hn$Og`h0RB>W01R{~yz4<#;Jt>2dB~^I7%*rSWgG`N;Bn zQXX#JFaM*Jo|1=``OgZh!Wyi_I;_V=bhYsRSF3-#rv5?pZdU&w3yzCq+uQO#jyB4R z+6OxF^4{S`{(a|OYOw>=$Xsx|J&?8;+)8|csh|o5!rYS-~4zCbMHMkNk0{Te^%U% z^XNtFS4tdK|5@oBA&wn5;&lK;EYH?o)g@Agfo z50u9Lkg3yg{-1v4FIx*M#`&c|;R@N0Yq)`~4f6kHd5p}-=e-x@Efna*W9)yP|1bWd z+xZ3ZVaF1AbGiIab~=vZe+I-;{XuJTl6UcV{Lg**;aPQ^6L zK;gH~g;``J=3+kjwmcOUlKo%u+|@bbbAJ=*&B|Hfnb(x>tBuX4*L+NW=od$ao_p*A z8&c{Yx~L3qXCE-|lMt3UZv|Fi4c1~E;(jO9Uwtn0{Zjvz<2e7X-#FHd^uAB3^OFg2 zR*~C{%i2cPqW-#f-)C$BJ?WmBHrVHdz6YtVMvC-GBethXXi-1m3(S&H7KbJh1+CqyB&N%+r65-dFVBzpVd$f&P24?Uwof_ssu4 zX0AUvy7m7f+oJ!!&iH^~`uA}ZX>F7p$Y`_7F4w=mXl&?u-S~hvj1PE8{OH}wz8qv< zNOMxc5%+V>w~-`MNTU-q|G|d*O8=%d&qm`4n$FsvK5u{e7JC-}apQJZI-Ncn6JJ&2xeEd$n(v`|_4?%I(IF;h*Kh*^xEDHc7$JRZW|O#+;%z2d#M$basczRCY>b2R z=3^liqvU@JUrLXA>h!Rg%jhdme1-j8^sfKC+WZJ%iI>!2$hC<6_d2qgk5C$a@7cz6 zcjEqH8-=&JpI8SejWZ;ltiRx#I8MKLgXdr8-L!Z|9Nygh|AuYobeuy?_g{zF>m%*I_19th{=W|YaQLsn4ix=%9RH}?-Sx_t z@SXj`!_Hx2!uMYPo3Q8V--MHX8~E+V!w+tKAnfz*zAtRA_x__v?oE15`TY#eq3;u; z!jCU&`}&x*-^N+PQN8xybYW9Qg#-4*`q90yVgJmr;fL*G!@jb=GKXz=*sIU%`-}cM z?9p%by%$D?-R&d8cPFh4yB4ht{l*Vemm8=5obf0_y|3xs*KB(Mxz;6ILE&Te#UpQE z;PX#~nlEbi_>z6`9N)%W+(-4_SA>V;fcCC|>UYQgwXyrZG5+t%+Jp|v{}_S_497?m z0$<^?`hMwSF&+~!8B;M0GmudK`6&)7dw(Jit#_?{%<@|$=HSi!(y%+28^(U>ziT{b z4Y{3q)J9MgA+-fq78K|cIm z{ri05^la<@yXJf>#A0OI#o_m*^xSp+|9<`O-{u$i?u-2M0`eEdb<4g0&Plv6lpRo4 z+G1WZpM8a}Rj515|8;J%#XA0u*P>NiDKuxK!+jhbJYB!kr$XCnpOlX2VV!f&9v)q}N+}^o%&pq3*oz z21#KT9A83fW~g#)XlVAImc0ER)&Cn`=6@_#PhXKL!_Kgd7Xf0z%_j-yDU0~ut|iTmPsh(SM|ia?=?n(oQfHkg*dh$)2?1b&q-SkO6%|aOq(FRfXDk6$NeDtp11zz z2hH)<&tDo3aaQ?(IsQKv^RW<%u@uWN@NHujp4UD%LVd~c_Fqg6tH?E|pK1Jq{JoZ* zJmJ}59eq7g^j5Se-=nYBFqC~{lS=KxOV}Uws|Bsy>=dT?&qi!Q6}F)kJFp8~3)tr+ z?DHb_`BnCrPoH1SKEK93lWlLa|2TS?pL?GFdz$^i&D{T*|E$h@t7;`(uZLf2sJ`$N?M$=qmlJ~HpPKo->jJH?qp5yx>7 zr=sqgIzu*0@qDLyzKH$s&1YQ~twXRJv~LS>e(FYHn^1*X6h1#Y>>v||-P3136XKq! zyBzO9Y>TM=s&NVQIJP2=!*97G->H+N$hgN)jsMhkm?Q3-LrCCAJRbvp>s^t3|Kd9# z`=60#$Q+6|jyV7CBzX#Fa1L>Ap}5BD1$t?{mr2I2(XZgi`UB7UmM6saU-QeSzv$jx zR8M^|G&^qD{$glFs@r^H|LMmy+`xP1{|z|qa!xmT=2i0lEBXJGA&%20f%GPJdr zpPn&4z1{ruo#v*^Q7pr3gLyDP&}^SWHkVPlDu1eoc=_7Lw%%7*{FPlGq-u!vbaI84TVS3*3qoL7lCn2Y++((j!4^yEzcec{EhkiHnH!TyVuVbbrsIJP|2 zIhXn^jxkt9u0U!2--nGmjr0G4aj3%9Vjb3FBR1hr=Krblx1h962Rl*aoNcJZ4#fKZ zE^-g{p=&hz%Z~o1^Z(w`{)eNJ8c^f0TzBZS}RE zx8~?G{D5DotH=ME|96SJf__}X4OBm`&OzSBUBvl+_sNHd>wgWZQ?_CVD)4yyui^B^ z>wk@;k3vcr&>G7D#P;{G!p37FCSxk5VFm_1;oBhlKJ#LzB>O*I7Pd@M7L8C|dhc_E z#rc163`HC}FyHY)Jf8oznErVF-%|QA{QsT*cTO7T_+P31Z7k0U`YNo!TKsS4|Lta@ zP{jYu{J(YLS&xm_g!)$>&Ht;SKc4@$jb4jCp8vN)SX$e`E^-gDx1I|7$OFjm>w5ea z_qRFZIDrD6zlh!w>M3N`tJ*Kn^Op8|^s=RS#4&cI`HREU4@RpWu)%5Pb+EY^Wa*tN z+0S|G`;zz6pJKg&4Q_p1*^O16-+S%H5%-WpD&EW2_zru;Lobd0`sWM&;Q%{ zn*5EietUDUcF(_mCQPJH#z0(KU;Uu(Idcph_y6IkFpZpnnqTw3t*!Ux#{bQ7ZYAbm zF6Lt)7Go*iH~#PK_q6ZRi^uq18RcJI`RDnh@yGK&mWd;_9jqW%q43-@VGTK8eZ7I7 zyzBq}t?>b0Qy=`HxiNlQhxOQqIDTsrS%qz=#pCgRJLr$c|Lvmh!=H}-8ydz7--81< zgwp!&{Imr9@%rzj_CNhUsS$irwD9YivBetuTdlMA=3#5$*L^4)@t@3L{-wG0NqP!t z^t^0*fbqz^WZpUf1+r*dbpyY&k$>8>lK;Dz|GQe;nTpVwxBfoa>AyJ?aU3Uc3TJQ* zUB)?gZ!rGAxPu(oYurJ8nD|jd+vy7JixvD2_eXX(&Nwgoy10?JZa+owT*4Lfqn?jk zchg#H^vCND+@KF2rEQ{>pZ<9Lf!o5~o4@ZmzK@59^Y7!ln?Xmsm)~f+V`rYC4?zXe z@=wES(*KsWS#7e(-P&e-GcCTI)_dCjQ2i5a^v)fLQ5cKyn25>f`g84nFKPe7e>(nP z*k6RH;+cjSn1$>-_7|1(+;;XCbLewXq!*Cit9=mLKl3*sjxU?-rX2Lr#gPa;|BfA~9X z`t(CFe2cb6vd=p>LiT^r^NliSSBc?Y85+`L!|U1~(1c{8`|4(!(MnIDy4g4#=jBku zah${{oI%%X@9_`S2Ya99AI|0mTRD8OQx>RcDEP7Z)7IHvCKEm8jb*EY?2;er4Q@=`){qc9e6ZQqRW@$_6f+j`iVH1x?R?H|0GjjdyA8`)3SY8=PL z;mQ1addW{Er~1z{%)l&EVh-kFKDw@If6%7iL*}&a_mX+X1+u7ZqO|`1boQHFjC1We zq^mUlUfMd#;{1E-|Eo`J&8S;pp*ZUgTmN79VtP`2BCh{m{S#y19j9X7fjUKt_?z`l zEE5*@6)LU&|55KXV?QKetB}b{`#d&%iTbzWwJ6N=+|m28_dqtduSRu-(mFkh*!0)g zcC>cO|A^(sI{#abjo5@LY(p)&8hw9=V@z}CeM7t`(2FR||2;4NBi-ox6KC8TK1=UB ztp0IU`@VQ~VGs79{ucXtZ*Vw3k89pE$;XH23A}IoSefEivJK8zeN6$SiZvE&7=!Z~rTyUHxd-eas{eKhfW5Sjt-pgw3f3J)Q)z*I~{!0HF zz3FA?TJY}t`yWcXL3%!@>~gJQye_ii#4+8P`SjfD z(*LTl2=p_EV+x<_f3VX248juR1@aQ=SC0u-$bQuMZ)sfwab0tK1Eu{B&Z=j)ZruN% z?+bqu2IBdD|Cix5c^CKb5QCcJD-`U3GlZoZ^5}oT3!d$icLxm|}`3qAOZc%wmdJOc6yCQRG<0lIws2 z3Mf<|3Mk?VyhOanJhkFj(*-z9Wx?{)NXvU%?MUa_gg^_rjy0r{D*nTbr(HmT%|0_NetA zJI-FKJ$J1!nWu!fF8agE{+)J??~m(0R`A^Le*7B!*U`A+H^PaNzY&g~d~Rsib6dE_ zy?mhXm-Roz`GbYw-c7#}K7W4N!}_~Y(pgyWwt49CMY;r3?nxzfUea+dm@6 z#kcS;BL4_~j6cQXA?>p&`J4V{$dxLykSS5uS1WTaSkqbjpW`p^mq@#wl=#1*w-@>U z4|z9E;Qi=OzbC|hfZlV`7}mJC3vvAaecJ!4)`me`Bc`{hFdQB9zd&~MF@5jRzwbki zNmc3J-(l??`p58bOsGRAF@+4W$f-*^)t6o4>c1NGe}g);Ro$u{@6$)$U&3y8Y&$-U z9`s`vqnN~#<`)!7zf}6pokb1<#v}$YRBNmOBlIWEPiS|poOJ#ce}}(EM;-gSm`zRZ zNxC-t6aAkN{|V419DO8v@vq|k4U70+xLQA``~F>c3IB=w|A&n${TKZwSlc6Sq4N~~ zpBC-^IqlP!FPF9d5!aZA>rcn_|1a6^di*ke1<%0E_%)188~>j*{*TFX#{XB1|2M1S zT8)*E1ADdqF;u1g#|S;H4VSjx=!)YC*M-Rq>%y4#F&^IwJ%=e|Fm%du)7BYCd5+#s zTvKUO`)Xv_b40f`R!^n(xYs;N^v!#3XX5yZ_c&{AqicLFegnUW-@F8mILXUroe ze;@JxcYi>pmwlfX*%9yP{q{g3gp>yPC9AJu_Z{U1~2-PrHXkjPxF`{Mc|`T6&~ z{#{3Z;`$@M@c+Yu+W+WBTz~I>NbAq>7x+v374F9SF}6efj|s%S_Me;oH>3TJA;fzf z+2r{njZrKZ+dm~!PooF@ z7#?>HGS#L1FOQGXCy_%eBPKEB{xit?r?+VT)tEP7``_a4@b~B#EDDK(#&PLA&6mgi zv!Bvd6OR8*>qD}E{f{Tj|Ksof&-O`?X?FO((i`;|o_Lxz7<+n>y+5UF&alI??C~7g zS;hX0K%7+p8K1HUH zpZ}-+{bT9;DgHbD3~Ap{YEZpFpKMjX;LquQfuEiK*YKkizGv2;v+b`C=bqe6z8{Gd z`EQP04cC731Nhnbe`DvBUzj+g{K8bN^6Q}M*RIQv1B3h*$N4Y1mtl-tbp1%XpHcVF zY0ST?hM!;^KS46i=}*q^pF}@CKs!E-qpQ~Y+u(ha@V^x9NA`E+ z9dZ(J{y~mB@grr3zJR}_{~i7w|A1ll_)p|N<6rS_h-mzxNA{~v2R5h|FjS(R*rZ;tZwhJWA3dvo{i1pW6VvMd z(?9KZ7oYEkA9PH@Ja^~$P5c&q8|kDpa2I{@f@ASJ^xs2lGh~rDCoS>6FRop^^#|l1 zqT~EE;g85aMuOhmb$MR@p9=qX^tHHdYKkK+?)$EUG0urBnFas0NQT>O)DVVE4nBy#9%`Dy)M zzV zV}u@kTd&kV|0az^44-0WlX2efzmrQyi~A4qKQVb;IZ&_ekiVypTTq9H`!Cz#f2I>> zgtZad>G}1XJ*%U&@}ROL)}_gWZxKE8SSI|$ao6-Y&kOu_h&=Ih@auRk7K1r3KKVPo?_vAgW!vu{-NY}M z{C%YC)2S}(LU)yGLw;Sd8rNK{{y*%V-m)69pZZDo1N;9W{s@1JKgEB?*hT9fAm7ik zSDxN4Z||1Jk*j0>7pwnE)&J!1S@r*%vK49iC>DQ}{jUuW|8KVV)b&sDAN(`v{}22* z{t|zMxR%h}ESHlTe0Wxut{{i_CbYzrSkNlY5+>iAwyGQ#ZE8U}g z;=h*q&{=mWbk$$7?NaDLFFPxVzVS=m-KFps()k!Zj!&Q+pGFV*F_yX%#xs}nfBZ!M z$4|l(GRPu_fwPywAcp2IY5%a{q?tmx=~5WQ@FqxS6qCpyy}>>Om%`u1xFTue@92LY zx4YgfGKH74Q7?sm5Z7LLDf|=p&*<29Df}z>Z%FKMUwhpbjz7m*PvZJ^x-azFFS+}Y z`K`)-b?`9^yRQFbpOouN&t3}uPH%jJdKVM&=cIf&C4XkftZ@9lJ)p6TgMu#$9;Q{A2ab)&9Q{*H`TOJEQ)0k3;0}y!=n*|1V08))s`{k;d=g_t8GS z8vcMx%&dlv+0}6TAJ>N03upP@#6G~DLf?*WIZ5^j_mk1*FZy1!Z(z^v=YL+ZHoVa> zZ$`%^dHRuczCU$ykvba50(EpLJCf{k%)kQwAPgbSs7en~6xJGrxiPsc{uh5rXd?|dDT=S*NG|6|(^;{Uq--`x6et8{L|?YIMR z4ZsABKj@odi+3aTAHT@9esy-=xHgEkLoa#M_ZPpxV^!LR?9g5IN!4k89J(gFgx>gK z_48|dKekP7bA3Ct9gwBxwrC@@tPd}>-^=kzyc!+GCKCEGUQ6#e=lu409sTv_zu?)3 zORjiUSa@M!c%%5mXRixyCf|y;Tj;^P>Kb4bHL2^WQGb zozmXw9Q1zje)~Lthww12jFUzCp+k7=9Q*TvcT7&M%Kv0WI7{ZLm(BQ!#HQ$v*U#`F7tVuRG5W(wIONpTXxawpISz?)j3F^6M0t zU!O~!%VA)j{Jlk<5I=1D2$`~fnjCem;RZf9pI;NcAgwqD@JnP`T&j6(_%c2JU$2wO zk*651e^8p|U6ZmRLr%%t4d1&q{Go5OeR^&9iv1JwYr|K`ucJfUH_30KC#eiL?cUlh zmjV4`ECar4+xPJ(9>db-*M`T*OIS1Ly^gF6>&e*uzLt!>R`oT$x4IBE3g3Vm5y$qj z>(uS!X5nWcj`20VxG0>kp5Ww)cjWzK65i3ccO&0s>9^SLR@{c;`afa( zi(Gi#`tTAmj*Ue7|E2Vox0ghtKIQ}?WIzPT&#s1#Q{zm>k zdu;B3W&YQF8?r~;g_zHdU zEZ=2l zR(Kc6zGDZJe`l2KzL^x6o(|!XV;ZCX-_v}1XW0jsn)7^+rRVH-rC#pZpe;Nu{UZ4S zN%SJl`CT&$#1{>vhRIKMB>`HgG;$2CA>Suna& z-WQ+Rtt=3}Nje=({MB35hRyVL@>%39xD{~?!aij|Qkl?;9(w#&Pq+Meo9(yb4!j7v z@DjWfFUKqKYP=S&!|QRie^y?6qj1jH%bUr!W62oMJIUi8T@&6-z85F7eHzzUQ!G3^ z91%W-g%8Wu`TyUh)YlpQYvh3Xa&TGwgJI-y>w=u^+f zvGzxu<1svrOIS1PTg0{4i2V8k>h`Pa4^*q))&E)be+~oE8q}9Cq^&Tl%`oB|DWq%l zPv8b=+=!d78B4}-79QCUo<(1Lq$I?8{TBLy{@JCkJtsd_)vo;fWJzIYJ7rCmD(x5B zZ^P}l1L=e6*qZg>Mf6E^?S$vMi~bViiai@-)a`Nper$hysO87urMA5iuSUN8;n}>F z-gDl&S>Rs~{}1pWdwTYA+bnrbTfwt^=boV(}jzCVt-=%>+$e9=XyOFiysRoc0Lx4 zZ+JX3EdMavbNUD21IE_gXxp1{@4|KAL&Y1z2g@D}rwXK(+!#Jm|CI1n+un}*&#wy~ zt1wni*QpyrC;FRjGoJT6 zV?Z~A-j*9edt!5V7;X9@9#@`yy5uIlCpU$J|HXZ(_2%$iX&%8bG~yoIi~DdthR@v^ zhA^1?jWBTTw(x+shmb0Gw!ZjVL#Ohqt4w)P?pny+?aG>zGAE<`Lq2TZ4s@drSIdua zeUids``G{c+5d<2uWw)zlw2Nr%hGcN{2!;4eY48Id1WCvA{^J|%lFSmDUIWwFAZ5T zuH*k1@^e^vneXktS^G2k2k`q_Y|KFzEt19(PsRxyT9Tzu; z&Gvg161I0IjoZ79TZH5P!}`ixce(qJev<4o=G;Z^c5OX_;vJKmbAQwJJF_|5>bTp` z80R0ptsvY^zXLDAE{yHh{ywdK#U!SXIjh`W*2cqtdl{5=-akMYeAWL;{V*!rz1?#y zD-AD^)=SYR+~536-=Y67Y%SHlV~oZ>8eVSOE77s3G`yO8EfOZB3|DRrucN;nZ^WCC zF7f>ER{A74v2$~HJN=!=Ex0!IP{y@Q6>JXg7WZBpLA(Dz93vC6rR@LG&`7@r_o92n zb&Bhqc3tIcf--G|?d;csm!Es|fjl1AH1@q8C*Glb_TzQoA^OAUKsWj@gfu1)eb=+( zXYe_E0bjzGvGlc@!&k`Y!}nEk@tFnT>*P1_ZG0D#HNNLV+8pxB_l0xHmMk)uk}qPO zdsO^mcpR6Iu9IKRushySsQt_SaW8RhP7m4Zp5j`eea-GKrHt}Ttx3uEbC<^nKCAt` zUU(yJz*v|3kE{KERi3T7HH&=z-}Dp4|LixSjg>;0KHBD=L(k#s@B)5M?~=ct?mL&} zO=uju+z)X4h1Z15!q37j$eL?>E4lPVz7e12Ke4hV+$MZG?!b%Cp*%{QTN8HCZw8X)DStms{SWa!s$8nzq%~?558GCZ^hg3PF(RHd^i0{{rO({5#-dTac$iU zIkn@b>;EP7o9^))?w9`);_TOmg!t}*?CQhHO}5nuW08Hr{lfWq_T)YGTY4YA4)Q+S zj|cD&9!3Yc(T5?VvG^A2y^#z0afWAH5B)Rv9KL|G`%K|W^vN#Irfq%rGW{#asqeGm zGMMswI>&$N|4`yOkzbdu-19GUo$m3gj(MVg@z;gFiErck7}J&u57Rr4TU9o!C?g8JD|CzN!w`RG zy#5(-;o0lM=g392T;qqmuOD9*z9jr*9A*1Ozqe!TzxH`;esKxucR0^i>0ie;5!e3c zLGMZJfAqP&e%JY3@Aq}Ke;eP$7w~;NipS8|YW{!hAK0mXV23epc4>b8KVSWRzUu=e z?0XE^HjI%Ow(lJO|7rF;yLW8W{B+@oF7wkdm0|lP*udJ2IkK~!{f}<+pclzg?Ei+# z{{N4=hD%s8s*Qkau@N`mMvS$x|C8+h1pA+D|D^v9>W3JZVgF+Y!x%}i|1<1=a@29d z?CYDPvl-9AEy%AwDE?OZq_*w^Zlm9hT&-)bb`NawDdhc6_HFz>{wI4khCA$+eJcM$ z@!b`k8;(C)Uyk^`GS98tbKLDYqI0MI7j#!$-mh2MN%S>&KWJ3noH!oBOC0-Byd1B@ ztMOVqdH%`I*!QRPfA|&#eG5b6@ILL2O5f&wFm*B3*6$f!d;Q4}DTyIbqE3Uiy8=XySk4dugu=+o#j!f7uBc4nO_mTZ|eD&DT^~;yn zf6Q9LcSss(oKOcenpZS-R{4wkf7WK?X=QK*dHAP zT^K&FCVZ8iBGdRf{hRnUCQfOeHn_f4*GFcB<64sKN&bDde;41!qv#Nqz+?0{cfNb~ z`tUgY68eSvgp*`%#rm*jOuodm*odq4(asQV5WW#NA@Be8tNM88&%!OZ6`ktfu9W_X z%uoFbHt5^m!Tv}8c7FGrYr}2!yB&AnMc9Rx;H4PT&ow^#|L}j)H!!%B{V%QIa`r!& zDrWzevj0~?c)2td_+7t}T>N2Scs2Q2ybiC&8}Vkm6>rBo@ou~qOW*O#*}6~sfAJCV zaqWQAY4P+)b^nAmLL)uPeu#4-uh#$S{`S=RaIbCo`G2n{2=~!Db{Kys(DzU8IpFzk zvW^Y?A@o;x{)aq&#I+h87B`$x-jm%(E7$wTA;dapLO!3|?|T!TKpgwYol{Qi@g1!4 zpQ-l!BR~Iz{MrB89{=8*A?vu$;B)u_zJxF1D;V3a?@v4X|GNIhtoHA`@o#eYF#ESw zJHh^G`Y8D#-_%#7^L50w>^I584*o3oHvPN!KDtZg&tlhCB!8ylH-W6UV?xlD+UWr%ZwRjy~ zkLdgVMzX$6KYVovZ>GmFp|_LojN4xk!n?`XSNUEt&VM>W9z$c~zul!Bac3BAEf0@- zUT+n55AH>J*8kg6<>5a1#(;q;tWS8=HSIF3B3J-~EpDPa!lO2(!^3YB8 zA>p`g>GWjE^}k6s##ioA{_YCB_DiBuxXWDr?gP6*4|f=4J&N%;5%EPyXzl-nVQ9Op>uDill$;WXCYbM-hn{$X?PoLbgD@@>8 z`bOmFS!CSz6xt8n6>boJBW}WGbR4)VB*Z_9-qYaO;1>F==%>fE8IsMO6>bwZ?48_B z-huR%UExLKE~Km*Frn_6yjT!kBK%V1*PT>eOsT6f=$t7JUFh~Md(M|@-+M3T%0u6U z^3cDDeSgvV9Mi`3&#;$`!N&P`uawS-UhU_pbzw|>Jic8Wy;c9$dHr7()y;^$K{*U4 zQwD40|K0LGM&#>Mx%#P${kx5wdbm8iT6(WVM_qY%9eI!MvE7)=K(x~htYv<93NO0`pB^v_dnp3XP;`-l1(vEA0rO7Nl6Xmb23!fvG{$*EK z`r@we1^SmT>^bM>Kf3?;?}0Ci`wCL@3D0OUrClKWb!1&zuG)1bT&HV%xwr{@6W_*n z@qIjs$BClCmV7Bvo=bjR#KQ&Q z<-)H-_Mmz}c&bMJBRkpUUE4ih*V*GbW1A|u`*Itrzh-@SwPRk3*WvYeBi@X+;_Vny zpNva$0@2rWij3)IOP!}!{ZkbCU$j5ZX@9DZM#z+KnjE!XJ3IfK(s?(sFO$EmOV@E) z`){r=yjNV$HhJT0VK_oRhJN9`E$U#hcZcsz+&#D#_u+~D3HJ*hS57=c=KcG=cwKmy z-hpoPq0`u3SBw7lvut~1O#Z*zr`7N1*QY&XpEM?r#b@w2d;wp=<^PZW)c=vc@09jU z2H6(v91NV&|Gww)800Vf|M+hCMLJ(WZr%0at7KYSs#5!z9@jLO*sEPa|90F*INKuc zG|5Bccg1}lkK!>beTx48c?oN#luJ+5_8=GjoBq*XEec2VSJxx<>1>Sg>ivaJn&1CP zef+PwCNzBXx^RQ|rI$S;+!(h%dri2BTztn(VKezG+=8V}b<%gVD?e#H$m`72Ke;B{ zDn229-$ve!4smyoFG8FH(e2ukRo+{LvQ0St)&5p#V zSerO8y)JxelQjfOOF{>_+Di2il&G7Z z$%jqfd()=SYs{>_;A!69rZ9N1SRZS#e$+#3^|ypn!P~U44}~siCNYQ<#*snv(|fqY zc~^_VCvj}+E#duzFAAq^JA=k9?d3A8^PHn$E+_&|P@R6CPgyX-f>`?#QU;InqW1ja%_Z$DKH2&xN`uNF>VaBnun8Q3C ztT(@p98))qXXG*R7mlCR6otXMqA;|VuLVZbSU<$;3CqA>$HEb3#;@= z|HPYE_V117nE&^y*6%BEpPQn8!*yW_1sTtD?z&J&Za|0e(IRpa5{~PZPEY2#P%K=6 zzDng9o2j?ux{ySvxGmU^W4Bm8img zq&?3##&D3HU;pEPa@sh{34IAq>Oa|Hjf6^Tu+EypcZM$=4mqZyPWfM37!K2WF6tj> zD-6~28uY6d`exMGWbZ;@s1-L{bxo)v>k;kh2J$4LojtKZy^mAEO~|TGbC^N~%bxK8 z@y%#KE837i7dnmI=lcgz`Uftka~+dJAHDya@%=9LJ%;wO?=e!vzDJrqim_Vr5B9i@ z-L7N5^U^c>_y>@=o&5hw)wj+wh!n=rQN{jN4`=8-`_#=iO+SNvZJRzM(TgYkfB3BJ z(>VS&>R)!(481WJul;O6n5EBQ9t${!^XS~K{%_U~jGl!4S#9&=ta3`*ydM|rvy2s7 z#3~B1;b#?wd{@QvV~azf~li zO~@X7W+*1pmA;>{HKBw)Su1Z=uL-5}Ey(${vbeH-qCpURB3^|v%T!ij$|Ui!9YhLr7B*54YZPfmM&$k0zCN6*GE zrZD9F4j+QIq%12^zPI-?BI9-GoW&WJxF&9j)s z3}!Khc`P9RU+;`@rULnh?9+DdKjqtP@eMa?|5lfVbM`xr3s^?Sq0*2zP#RX~J*SNQ z<05?({q#O0(Tjqd{6D)n6p|Z|Hf~--ZbHiV`9zy>e-sOsAe-16a$UymjkRa6TbhF? zwe9#i{W9!2iJ5{%9HhgChP1#jc~& zb&ylSnREOPF7Q7f+na6bJaq*(vF^X#e9%|L(vJ&j&*o#>k5NzpDLDjykU0 zyRMN=E$UE@jv4QNt~4~zdyMI?y=l!Gpkb1RBd87Sv%otR&eOl0p zHpKO}66A@$mUq9mE_Bh87(@!44chXzBr7L3jL2rV-yRoG5+3r^JV{<=U9K>4Qs=k zbQhiy!aTY3Vf&Eha2^-1jHOrH93pZ3trg*mSViF@zJX%pXU&?hfqrHEt>e#@cfX9+ zy~?P#Zf<@ZT{6GE|2g*m1?3nyvZ~x7udMGsHqZXAR{z%gV*7u1ZhhF{JZ0F4a-ul=Iu2U@r|z0?nBG&*{!WEtUTZ8r?%%iDwUI5}0YHWZweSNR7O zl8gFhHjqWwgkqGS6kD(r+prxwu=EP{L#6&ddAM5}xd%Tx{*kp#Seboyq8z&s*D8*A z>L|uM!*S%FzjrX@xnF($^Z|MO|BQcl{=SEl?;+p*a@@jOv_F2p{*UYb{LptU{k_pJ#4&WfN#{LhHOK;Quc#itJYfU&T-1ruL0oD<$rgzloS49my&S~tfSQBdLb%<-q z^(|{-lD)g#e*^pf6g!#B|NoDkWc&A}*vV}63dhu=0Vi<^O=w07#*)VO?K|O^Npi}V zai)%qPUhJ616}NY@k6!`lOy(zYy0Q@|H=6OzgFqAA%QNW%k5L>I_Q(^`H92Qpr??l zFgALK{eQr@$e4HI|NrviK=h7U*JHm762{lNx4Ew}wf z{|Wmg?ML>y&ZK^WzIppi`|p50gA;Em2xlC77Sou)Eaos)>-}S5TK+rZ{m*;ry*0@^)KlEZ>49A^C!rOas3mRN-Faj z)CH^h(O1^EN%vpr{&75*zwa4jpY9pFi+{k6*M-G9UBBZO%>PT0gGgZ<8Jxx$oW(R| zFpD`X`5!=59$$EuZ$voG|63r>;XE#&eR)k-CRcC~tLQi{|1Ycw1%K=P*U0}Uq;J60 z_M85{UgcfqHuVp>%YWMctUo=D?f2EP6VN8T*vEFIf1pSj$9)f*$YPA0Rlg(F(UX{> zXOLBA<}k2(*JIQh+8r;`O_l4cUd(cPkM=z4d_rD+bY5iL#{-F57?w{;$ zT_5&3W*;h0i7M>J0UX5Giu>1AnIL~|{owllhySB`f7JCKlExK(!Nc_EBUw$>pcc8E z?std#CF6L-mGO)n-b{B<0i@q1RZOZeV+AFp#eQkXxA~&HJB`C$x^UZ;M z-jT3itbdDeTpMj`l|9VWKiZT{cgm&^B6ZNwGjyYhaZ`tC4@ zAq?-iJB;l1j`rOh(&cxD(K2za>#%gHQG;5fn_Xkwkx)mUOu0AI(;JXG>E0ULW0QML z90@1IwU-_Vr^qIB%p3{LWD62IJ=bN=6|KT;=sWGXPJ7OtO>)-r(S{t899Zx|!= z)Ud`2~5_hzYeOu4!b`x=UYukvkOVo|E>CkOko@ubZTpKq1&_QIp_a^ zo=uYML;q|Dr|t7Z|Kc;kXEBZJYxEl*RhNBlO_&v)!#oyn4q0RZJ zj1^qODhkf{{;>f?SbF40*hChi1abavDY*q(u?<(||8A#G%4ZYm`|Puo@xoW<|E{q0 zk#p+b1!XhEnXtFRwS-}>cnfE=za2nWeSIE-q(Qm1}Gdi4G69+&T%V^t+M5X#~ zpL)}=aop@QIf(X!g79$5y3kg#E_`xXzocWv5$ov5GwP^$brqRKZdv_Trv5Ki|C7T- z>i=SOd#U=L9Nl2w^cCqZnvv%UrIT8Cy1tl_uz_BLO(;gbp4;I2SzQ}SgxiY?Ln+z# z*X&1a)vff7t=hb0g<%`L=a4>>1BGEbeFyrhMy_G5Pe z`xsaK)7h6-{r|VJZ`rZKL0baud>tjiHz{uhW<6{}UTLUv!tc&SKZK z#dV<%{n+i8J=lwVs6ZvEupeXEaO1P;820=mIVGH_P!}MF0ci|kXsh?*J&tViY@H|V ze53j&4oKr54xwFnbC^sdir5hT^Y=AAb^7mX#C4xn7NM8VK+-<|`_MnnpJk8dzE64L z`M3MtYUA;{*M<(%(evy71oiNC_3&2tt6csf`_Nx(&4DU5K!fe|w&Q2>U%fOMa1y7` zgl4p$6=Mh4_lLC+$jQ3PbN({IS>fEg_FsYxkkmgw4r?!s&{M7InP&Y1`^9Zo6WXMc zKo^oo7s;a-r03VyWhbZTJ3~gBBCh!u`vsyM zJu7Yw^N9Wjab3uTkMjS2A^88twFl1Gb{-e7j1^qODhfXD+s2Z9;iVs49~$4FeYQ^d z)~v4oqVlbVO<{Wx()3MaG2+_O6IH&A6>Z49z6~f2u^ijBJs5x`6#ZtNzy}7$S%5H&SbS9&vr1(F4Zc zZ7Y+`PLyLeI?9!E*h7zV;-dfiUiv=tXY{`#iQa9Q$O8duoQWg6DFp3&!w7bVzvJM^EQq`V$Jv~A1E_PkjYp`)qfUk^mFp(DjWTR{Hbl8 zL!oUOP=rk=MhQx>1)a)-E?g}i=t=aU9|ODP?|t(39(lP^{;u`@kuH(HF}6+lfQg;X zk12Y7eInbUFHe5lXOpsP%jIo!-+Q;|NowOy!x9<>bk zVc~tg58?O^pgOW14LFHm?eSA&6PnS2=-ZiE;cG^pZ1Zi-sSm#OOnm~%uOvO892oSi zvIjbiiFcJ4Kip>g5c&E2Ta6zY)9xpCIOdrDA=l%maql+aE;RZ-(Ft=Y$4(nx#{?!Z zg$%ODVPL=UOX&qN^TavoqiAn4UMamnq%e+-#O9DlNrxWC#JgKIhtu>k z=)d6F&N*h;wO-sD&Wamuxj9UeGe~D{4zuJOQmLE6gl&_U7hXWN*0?{WkU^(w>Z)*# zO80+AnzhnKAH5&v9CIEQu#6R4#3~BDApg&~f7dkOS|%}d-gP5u>_5jBV4%ePV@TXE zM(mS9nm&s5`O9k<BKlrr+4|e)OPzcl$Q{T% zX9pYq=1@joc;V)-lPt$>?7?2_L;Ey8G%}%WSxHu*L)?Dy0OG%O;=h2B+Qq%-m!IN4 zcDl4dJNY2=6xhDOcCxp`c5UiD`ti3FhJ%hjgu|%Du~v37j$&MyGKTy*VPpnV$RUft zE!yCFJ)^DK;7D!bgD-BhSQ{LjuCZ&zxzSUkTrFlJ7qX2D*vP1HzFO3w0rB5~C&^Q2 z!kGQXOFidpo-=;-{|@=}*DLv7?dNMnj(8ubGUb1{^564sG^g;y=L$8miC=`+qT@BIo-&bYQ|_B;Iy za)Z8sv$maftz`RLVW<~>7SotPN1^(+bbXkm_o#cj+3j=mdBlI(^|9NN>fT;*LEIDj zkIo67#|11Szy9}U)eH2CSVh6#Yr|jz^8S77{BHE1SKA?3sE*fu=qHQpvkAp0K`FLi zE4E>5yY|Pl{;@Oq$F%pR=vUYOYSsSRtFO9}e_#b)Ep`9Z^@+D?e`p5`t7ErIX9vo# z6VZo1)wDj8(AZ}1HzWB*q}3>%L+8Nt@ z@{0dkEB%W9TN^!rpY?z15|_jv+E?XIGI7EDKep;PJ%iKeRu;rEq8G7@h-E`Rp6Fk2 z#(rlpjqGc#3$tY7Bkb#E^XZ|_V*%%I9v2YD9=hc7ZuFp6-cKqA^6M|}jP-9(Shn8^ zE@Bl0U-bQA1ID(?|MJ__@n88Wzy6DN8T~x+>n~Tx|MLFuHu+zgsjc!qMvK|@d=87G zv&ffl6IqNn2cv{6#qqaYAGX9e{|VblZo_sgX$$NiudXM*-Pq3;*${8go)#b5A3NiI z#tO>G{QnR54(y@t#XeM^o$Wc}o_5nKQH72ohX%)- z#3?kP87*i<8^+8rA2$bKqSAQ|Ntf?I#$1Ff--Fx%{s(8xTgT9xz6OlYQ%Kt{{{Jr_ zjW~}zjV}68ZNdEd{Kkcbi=|g8J?UhTb4=1cgP5>wa)bW|B&8>u_y764Z-L$+E`bca zXP!OF>>o_{{@Q$h7@?<%w5j|*5vx==n;H?Ghp59l{iFJGju zB6mo8S6rrs{fmNW-w+@CLUIE-cJn7DHz84>Ufn0(s&|WpOOT&ew_QF~ZX{8Efqjgn ze>4YE`=``CThaJJcI~UyhHdoi*nu*{b-#C#i{{Ojle@78dokRk@19JJYya1+4HfiC zR3WZ;HHjEEwkG2I;8n+G-=j^6f-k89@c;F{t&smPgt%7B&-&lSe-dtxPUCab>t8Df zMfCKbZ)DH!f!Q6tu?F87wz#JDn)RX7wk_ybF-A;oLtI0kdq%y6 z?ZP|I=evvRTIByTdb%K#iQ9>C?8YAK#XeM^5>?ob12~97IE?H&tiK!X3=5w%PW`*a z%076FoqUUk-{g`$u!i5Y{i)g%_Fw#$=Z3~tm4_3?*lLBBK4soWlrL5ekPSGAQ)ogn zj^BA_XdzqCh6EOkM|P2M{Cl`Ue*k?DDU2gs8~cFR0`zF}PwdbqPCtVy{azV;k5jS# zv0Pg9@_&o`-y~0H59I&=i!`q)7nhKv2?kjpu_{iLI z!bcmnhvu^9h5OGuFMKSuBm7Ol^FvGV^TPwpW#Qx7UJxEU{DRQhQWhR6e_{AU*$YFp zYe>)TGUn(y>BBR-LW%Q~VheI}yF%9XjO|mHq)(jR6}H;;q5aPeUFSCFgLq2lDb!D1 zq+i_JwqE~i>)&mhEA!OQw!qiHzU>9;!o&1LtNC%w>%ynUpOPOZ`{c^b&{k;uf%A7- z3#~kCx9<*=VJ8MpKP?OxJIjxg)jchwW{r!X+_v2~wqYo>m zmuw3SxTj!y_`t>OVV`Z4xVKkZ8U$G|_wL-Q#zLW_kkpJ# ze_)IC2gycUjFh){_=ffzcJaNrqu){Hp zPt{laH+(GUJ29qy9*=djx+ePSPZg@0i!S@D<v(;GPwHx zfh7B1zrqlP8`%G?#vim((%LJd)#9`Ts-&|Y2XGK+*OYSYhv<{))(O{om|l&zreU_q zJyy6^ZH*dn?b;)?WF6wa1MA5KBplnV?GfkopA}f1Le~tN;Hh zS2l0;Znvwa$tK4>tm-9tJ+FgkM0E_?rn%v_3zfCeeLyv2F zXVAW&eH-`1I5LQ~d;Is;X?l5cy z3H|zW`ov|QR}`ilH-lNsVIB)Ohq(6YSgm^}C$!V@{aSPK*co*&20fbr4Da>~_PTd+ zq{_XMY2owIxPWD>;3AgH!(Sx}X56nawnhKkDx@#GZ=KAgzTO}|Ip+p(MHoJ$eMTPt ztmjv%|CL^fRIzK?pk08*@AwY@vh$a(E$=5jtNmubt=NX`So)tK>>$gc{tEe9AN#_H zcCP-HMe|B`iZ91*?7?2_L;E>>6^|HqZYc;A!j#*nO1epL9j_WH)i zO|~7tK^($iRHFv9=sd(OW+!)}hfUj?VgqNcS3X$ht&4rlrXE~jU$4fw{Os!s?BQni zbSwM%q;=)k@Z$~a^Hb42_kDV98D!}>)VHb+QRiH-ja^SR;3Q6=F^)}XFE-Je(SlaA zMZBZo7PdCM3rP$jz27xN=#v%BUFRC;8RT}b<@d79cVF&2%Auh5|m;K#?H$Bn3$6X z_-~lB^Nn6qCrD!(wqv--++Fg@`m|;AohZkIKDp>aGPSDSBD08d**ccjgx$99LG(G_ zOSYd~6ZVl6=*y_9PHAf<)Lr!Mly){*XEMC1d~k z(H*|Ya{0ZKJ@Z3$`r9^!>A27Hd`EHpfSqiF>-hSA$Mw9SFnmoP=#~GM&f0I%Gm341 zIr=)ql3x$G6@3fxI^voiNp$b_{ISip?dWV#|DwA~J%!%!%l*=Q^nUEH&lBsPGU1&l zNA_LXUrF_UN*$@+V3M2?&gg5;B8P!C_CJO&j1k0nap{Em-+2$Xx7Y^|*N>f{r_1%V zOM4GuA3(JIC(-y6_A&b>OV44d+IbLd|GoCzhYD1pqt^I^ZB_IhX~#Z*{qzHf|J3P2 zQrdB?-TeIf!}dAo{!fW>P00rP(EG`QjyZ(Gs74KHQHOer*=O9o6OOz3|AVA9Lc(>{ zvHw=x{~7l`?f%J;dG}AIg-6NZ3&!@Pa}uY}gmj^OHmwiM^!)tCo#v&}Tk*5=AKS#W zdrt|n3mv;%KY84DpQvroD+#P{uYU&6g)()P|4_U~!m1$xknr2N|_U-zTl zSp89qJNDQ`ZG;b-fA76#gimAajBV3wY;tm5yMUfC#-Cl$PH6q<_`EUtAq-;#@!ub5 zjAD4Nwt;lcVj43@i%VgaKG`5`%+cqOqi2z6b}lT4YcJw6L!L(m{Q|j+1U>%qrw1#- z7m;6ANVs=K8w9aVUC<8^*V$i0>^EPv-{L=pQ1CzHIc&hv3(cR6@`vh7vN-BL;(wwp zi(|Jjj~oxBp+tNsvQI4yTgk>pJg?r;u#LVQJ5Yw5D97X(evAAP$N40V@kh)}^IJsb z?BzNB!==6*3}OJI_DyS_r!a!JCRaD&-2P7V*ZFplL@#zb#~$p(K2)F*RoIWt3)c5H zHs8I&_&4(F;^g8L(qXC1SeY(^>I4L~2LE1Yvhg0;pet)jc z*gEp-|8FvOt_{(?YM%2R@1cVKUzP3Zg(mwa=-u}1*|+)f__y-BPg?!xHQt|J|9`(R z)x-8XXg~Y*l1bq{vR}H*(rG~}+G3jMLK1_xI{&s-I}cOZf*Iu7d9(6=gZn?_{>kB1 z_fMvT)8weO*zg5m>5L)W0tXk>>G>&kgF)1l`k-M zQvSz;ygnK8`{O)6X4@OzqW;xZETwlWm=_^_3%%!%dpp3d znZ6DERqjn(61~-H!*+2yP==i-$HJXAhu!4ji>>8A?!`XD@%Rd|5>?ob130$TJ#5le zPzD?nUi{YDaEOfUh{NQPHvI87X)|db*jmaGHJwxm9&2;uv}SSA5?7R=>o5 z-1RgmGg8Wq8D+>>Wyxv2XpT9HY0MzspD^fsVODsI58pT@Fv+H#I>`Pu29PClrRx8a z$}SAC&414S&pz$yxjE^~Bl@8&kR275{r}F=ZP?skpr{>&cNh#cY?K%KS9PF$HMn6Ljmi!1X43%=r8#|9MPf0K6i z;aOdEqW4$sJ1K;cw%B4zOG;^rZQ5dsEw#|5Eyto|lu^c_qB6=Tql}_5Hdt&^?mz;h z5FkJb0Rp5UkdTBVo=%JoB%YS+eePo{=X>Bn#ORyBn&~@L~kVURQHd;?e|2TR^ z`c$5FT#!D|RV;m?ZuVcz&s(VcXKtD+eUmNRHOyAW;7=a%QGn*T%6}Bn+w=LI6Awl7 zVstZiF?W(3h4E0rt}MwMUa}kqQhg6u6Pf9Uj?342yaRI+4l~y>SId8D$dUcMqw-t& z%?IE7TCVI0O|0@5)VQxNSRmv@!@!TFWJ+&@u$BST)3?tQhFrm`Dpf1I@c z+!yTwXneo#$Nv^@9#Yr4U~I?r_A}D|DRJPp=Abi2b<(aVZ$NZbO~sJ?({Kn?sGg>6 zi^HhJG4DIgeVLer*_ea5n1|N0=J(^Mws3o%dGXrKo!Z7-=x%a8z{$bRh+B251YtZMA zMXo^eS#u4@95m67>a%FqS253bq^M3Q&k5 zM0xy3|4*m6F3hbnd_Rt!@coF^By=J=GHcROqS!p+7PuF@~o>-$gn{sG_5zLJ}UbIJg+9wT!<$u>m$ ze2$DuFZI8fkD-g+z0kjp+Mx6bbNDOG>pwJE`bRZt5bg0iF|76{O{=sv`zsZJSbMNR)z52=A+rn+Mf5oQbSBKO5 zK7(EyDAiVH-$x(Wf7E*n(9fb)ANb)6?}%s)!JaAha1<81&yEQt>Xiy&gEg-{C9++bMFXSPF-u9>)Non>FQ9Z4wzdxDdbI=6xLvE{pI1D zXAEK0f@{L+dDnyuv)>U4%C8I?v8nA!neZz~=Usnl#^ug}=dV2@i^QX+ms}dQ_g)$< zxHf1%|1Bpj4qLM?4!h-Z{f}9{!oF{Z{Q`gPUmHK(Deu}PKkQ3Z9kS z6qD&w@OJuFH;v}1mwK;p>@zSG(=Z(~kckRyjLHGy1AW>WsHTtX*L%wN0P3{A>v04P zXjH~EAzH)Va?Jj}iXY%0(VM9~H-y|EE0z(OJLM z!~9OyH#%OwNt&^LZVIr*+UvCw^~ckTP>d41eg00q`ae2~r7LvNyHO{d)yr#-kPT<` zzsZ-Qyt$d)qF<(Ap*iI!$LU$&=(TR>yW6$iO^#h&c zF&sE!uM+YE%BT6R^ZFHstw~`%g^_b#R_G5JkY|t;7nGM6S)Zf4ub!e@oT`pYMrUuH zcF!5~q7Q#v{~KU#EmHoMD5Fc2|L9nzZiy~*qwcJ-7e~%3>y^EYy~_V4<$t~MA6+No z;Vgg7;XE#27)fWOf!`VnBcndNRI>jz^CQS{7}-yF;;ZrYxt@*j>?WXl96wj6A1w1O zv)*`S?(?t4(_Nd4DQH&iHz9+*Kk3zY`IOh=2hO}6KbZ9A_*8bKqAuyhc>Vmr z_>sAT@rGr8h)?6+bj&~|W??qwU@k`X|Gg_&zoP#A=PwHL*v-cREJR|u_?s#Il8c#_ zU@4-u!Qi*_f5}&uF|YBBYo8d7XVDMHW1~5gE9g~A{aa+yb1*WmdV&0No^M-Z{!hVc z@yept;)hCJi&v4=^{>TiI$w()u6!+Ci$(6qLp};nh$0lD1g*lZZR+dsqhx#L>+ue< zlevrR?s+|4$B%j($$2f_fJQXszZP#^@_M`_{q=Y$Kl-lo56J$SG-zPq6Pn*MsV3WOKG>tavScn%+M3wfIqt%+IS9C(~Ywccr`* z?;QVHyaQ!LugA}D*NZ+3;6Uf=@$$CU<7eqr=U$H=!a4d7YJJn;v9sKOfdZUgP^)=VY`|Eq;KV(C%{D*8}j^>EB zWSjrpG#sDCpXrGDU}umAP7KG(kx8#ozC?3xXVbfkkJmCsb8l!0=>ZSsesWcxyS0@=yjMRs2>pGIEOkR}az zRu-bNa|M}=Xbs>|X{25H$zjezwDzxip?Xf1dJa9GT>%PFgknVPhY~Vc3sg#$qY{J4 zoss#vZ`R*y*p1Zt%%xs?mA--7l0y@_HgrbrE416mzDHk+A1C`?64%KdoWg0GK`;8y zP?!`3$hX#)ou!|{5RUrS?czhnQt@M0{(N3qIII1iq5PY#oWMx`cad_ZL>Ws)`-IlJ z_dG6O7)ierw~>l8jLiQ_(Y98Wx04;({GH^;_3D1Y!DRZNHI7rrwfbY%s1KKU*9_*Vn1<;%piC{t z40_cH-?mIUnLZ2Ap23GNm}4+xt%3SVMWOGEX6W3 zPf>@OW^MsJ8v8iP%?f%px|iz9U;M_{N5=|t4cL|W);uyF2e>aFqdka6<{!8gje&HD zlSS+gPxf8@d3C1mobKC?D+B7){mDa!+WysK?Rn`O#qKFVDauia8q}i+QQN=un6!09 z8Y82&e+Rp$?cX&dZAxcRA3(JCezZm}TAv>E0kk0M14zuV1`n&G&8RP+>sfnmlg()3 z|H!$G`qQF4K%%~Y4*qsVe$%7AfHL_|CwIqi948R<0Yq*89(vVrc?V9>PotI|^#MeC zj7EI`XV~?k53|JYLF0D=^s_jJe*GpxWZ#CQaGtz?VWd1O4Ir^p`JAOZ%GQ5i9^;yq z^anha7SdcBhw+$zbWFw+WMC?$VLE0Y6SFWIb1)Z!C-wims7-Glpn1&wSJ{JvT!4jG zgoXiYLCD)18 zsBKM-?2|_(xo4Ge{hWbI1@Xf@&FIMff3vl-=W1t@9q6QYk=-lI4_K)E zdqMkGd$yrm`@d8BmyFigw-jm5@N3XF=8^d*Kp~1yj4t6{LYAT&mFPE*qK53V=S0~6 z|LIL=LnjWfD?hLAf*!3YIHWy(oPGk)n)<^t-st0~Cf`1PZ=UbR+xr_V)Bk}U_ng9M zoWWoBuk|vwp3?ruQM7w*N1ygTy3mcfao!I{(16Av-;ZXrpuxKh@MmQHZ>#wz^ypgD zM{R1Mk4#0mu#WmpqB#s{%;Ru)*&E*# z^_@g>7~XFGtLs)3Nc+X|036Pj2Ov6sZoK;@AliRBot%s*$iUm}fA_Q(Di_KsdN`ri*u~++_Y(A#_&s{^4HMabjhUu7r<_X-WTV&GPz1LC9qR&QjrgoR> zovue`ZO>sB?SVI!oR8@2+Xdu8l+Tx^t?*q~^RP13w?uo2)yi9{$r{)HZ|8q4;?H6% z!BRwL0VayI1LXm$&PQ{P z27fBOD-W}y=R{`h|E?GH|G((`3g-Ta`lHCGJyZ~x?ZFuBKU_#JLNQ7Zt!r$`(kDuP zYyaVLdL_Egc(+sDk(&-gXTHv3UxRuy;efe%<>vpj(MR_GA2J?EKZaUmQ8b6XhMQ_6 z=E?sS$p04Ui%@4c&dpov?@!Q2*8e>s|30UFpzQ5Hv~N$>DeeCr`}%Nm3a4=fz39W> zgUMka((lm6Pxfz14(G@roW}(WBWn95{Yrj}RHWg+u>E^6j$UQl;gGtHBb8= zbpy)(bISi=<-a=6NdL%j<^M6|KX(=WMWugoC{;Th)f2SKahP6<+|T-7Pw0E7P~TAo zP4~lM?CKjILoP+e#aAP%F}yuB6NWFrT8$VUMRQH0h`^Pd)* z|D+slCp(nGon#ktH(95yYu_sQkN9bM3L=j^Q|>y&a-GUb~((CNAuct}vg7 zeGf*;-%RNb9o+PNZB#hLzJIj!F67|J5YCXj=!@*F_Z*1yA18&gb!HjO_pCn`=pJwlS%rBc1j_5yaI5A27J}EREnu-n+SngGEi5X{4`?$#0A2L%QukP+pYA!6F$EcDKE;o6YtZTK(Y@#GzfYfzZqMnO z!H-OSaW{iqCT3wa=AiGhlfqoGzj0!iN6yCr42JZukX(etSc07|^G=|PyVy>{8FaDt6g#LSOnuKdJJ)gd2pL$ZV zIG{{uukp{D{Ci>FMMiruR1{19IFu&+r}!u7{tbB;wbSME{3t{bicx}6l%o=@Q>6c? z{GY*pbY$=!UA^M}Y4Lxq^v}%^ZW_qOOmUNJo-F-G&*Dctnh>2K)kc;VOV{*H`Y{~G zA!)rz+O0mX+#G(RZBWa6!nGcpLR9}fO`bt7`Y_U8D1JrrV+NSd;vCjCnoIO-@<{*R zdFBfk#^Cej!9OQIM=JVXH3y!I;{P~uJSHF=lhLrq*glzoshEbh*8fbWM{WM2m_g4( zcb~jLp77TAcU_zE8Anou4;m*3pA_MP7Gc#&j-0QyRQN6yzKet}-dbOo_vuk#7Jp}B z4yH*@bIEy_kBVvfFJ{R1P6$6#pAmjIOs_?>XHQ?^+OUAz{*G%yw8z52NEdIqE=vxJ z=}Ry;Ry-w_Aqy)ayGO4z|IPZps0^MJL==78?B81dlS8js=-C)K|K~n^_I={ku>X!~ zROD#`7HI#HRmI9bWm&W@?%_`D#Cr4d_>qTv6rd1AC`Jidm62^V%D-~s->J&?3Ceej z?EjOZ{eMdNha+gf|9U=Pv?pLGf67sb8Z@VB|0cPQ-aek+Xri~Fdxf%fnKCv-S&L3~ zefyHbF|z;D>LY2wV!HU2Y3!EW3G^WMv)YVbmL^`3MmOsxWH-3O{{ID|Lhfi~YtU}n zU`_-31NrI?IitcEdR39S0eb0usMU9HSl>ZSp85j@*fpHJD4ZqFp_x8Jo<|e?t^U6Y z%)|IE{eRlyl^9v4m?`|H2>+?Ve~$2DtgfOL%X|ILy97l@zAosJda zC%VMrk@NqK|G)JAaW|PCQ;>l&bIYcZ<>UcOqi>r0`}lP7P@4FgBL1d}zvN-&T5`KF zwWk*wTbn&DY%6(Z*t+bUVaxpS_E&r%zInm#<9iof6!uI{2_>_y4!f6J9d^y1V2sXu zmIc>_9kZ3KpABIK|NB35f5;@u>SN)V)5@i$QDI-cIFNKts7t+H-Ew9)QZUol!Mj3Z z(v8*@&J105j1841?hX~l?haMfsU7Njul|cWpe??d`3MbC8F8G?%FV7so;Yy**$1sZ$$)UWAeT%jU`37pTh>s{d!Cg<|$4C`CCA zEVPbbfqnsc)f{8P+0mYF;xlTC?fq8BOjgfJ`=9IoQ_ZE0+>y;E^aIdG>i?tV!OZQ) z^$#3lk1lj$7B}@chT}Ma9-P8yRLnNUon!rh`>Rlmnkm*Fct$N*x5)Z~rPd$lYiK~@ z67&AioNoQWc>bW)hPe1QT~hXhwwj@|7ps9 zabitE-Cw`Y6z(%H71J;sGtip%SL=UD)PI>f3w$5h-T7wyUzvAgK>b9y*LY6-w^#jF z-Jr$wsQsA9pIMlVIcUxke~PrJ>Cv3KXz%-Z^!XS$|1Lux9HKQqT_=-6nKZV*wS_ov zOnZP_jPg^;o?-chGI|N~QXF2Y-(Gnft^KK?SMaA2hi0h%qnciW!zJqfMIkJ6PZm}n z8#!22Da{DC1X_!Ie~tQox%&T7-=F3Cvwc6=&0Obu>(BdsG#vN+zTG)*#s_@A_CVib z>fqYdQ9Zl=KJ9G&=cDV2wjSx%M&r^J7y&F}(!QUOj^kYkv=EXET?h5;cg< z;BD$tub@YJ!W~_rPC?JzmmIohNaqXW(NSKl9?=%rZ%7WEkqq~QW8`t1Ko1Uye^DH) zXp;`=#l2zuH+|xs{-K)lZ}zJ#*3YDmr@majlK!q1=0>t9OW)DLs9!2Aw32P}qkgG0 z{r+j@{-uS^^JyVE|FzI_PT@4p;DEWvQ5n!nkJhdo$}lgQK7i;FuS;@VUkT=yT9rss6#Plk8}+?+?3q7#Cn--GsvrXM(C{Nj|o-sn}r_)w32etI6FzWb;TzlMzZD)QMi zbZRG%g=ki$7m>wiq92{FUa-)36mu!w>KBOippVYMs92_cibJTve>wlA+FDrmRiXy< zXhIu0(YnO=8jhywpGQZ!ydg!tkf#5StkeE~`}`a3-rk>&-XdQ(#*gDTfii829=J9l)AD-zVQRw}5^QLpYEA zFOLe7#mT-eT6@dy)PVRKQh0-c+ClVvvu)J=kE(W^mc3KkJejDPCt$A1>S9*XK>Sj zig~^tBj3M>zo^06=YQ=>3TNEci#`nCEY4vFt;)hSL}g4n-md@CyHPj%SM`78Wn-JN z2+i~sb|Wk*j#cecJibgoc;9A{q%c4eiR zVLUkj2g)-;Iyo8THR2ApRdemB$((`1{EE(?sh%$#XU2Stw2LTz?D|$bM00+oa<^8W z@tXURmCO1(nfv~1?JJpyS(uGEn2ULsk7)m&1>{@%|16|0!eab?-Tx=re{hNWmtq;R zumaKAzijgV_x?W*J`noVCHhBO$N%N~Lk>UlkdFcsq6oz(L6`h1+81PS_x+)qd1U{e zCBkr#e1NQE_tySDHT1Xl|M_0B{V9^Qi?siTwEt)O2dF65{#2HY?0?gyOrh7{urjC? zrTnNz6WY*;V>pfzXkBLhKSs{?>(%~0uKll1y^HJ~*8b;Ly)y0y{>T2e%HMg)Uqt)g z^zi2tPU8%kGnBu|xL$huGUe}5^$+?0y0yi-l+B$Bl)sDAKiEb4-<%_daDdzMQ zn2agNz*O`9n%*8y+$6#Y}SU@gBg*N}&^Dip($w-?uI83ia zG=F~)cZ;zEOR)@DSb^51(!X}XQEkX}bkIA~665Xqp>nlf z%L>#d*{%7r{*-9F#*n@O_2C-z;c7C<3!**2inu982})6pO7sis8nREmS5HQJ12vIt z=)^G`#|iYHVaV7%InDn)O`bvXS@R6~;-Qz`ZXVE4^w9^^GiWsDe&X0Lvj4w)Y-Iocv%=7HC&2(XPPZV(knJd9PXS zn~gb`i+Pxj1!&aCVMx2=w023ab_&_j!|s?p z{G<9?ll%({`5CPZXwt^%(k5%4%5UU;J{}gi);)p!IOE?_JS)Q*0d{4cwPu}kYLAL& z?)(yVOHp2~?Y_i!VHtB44$t>p+CtS!eWx~lL%u!N*k>aLd1%h@eD?YD_9D+OxFi(N z3((PWZbmAC}qasPW4sV~YK(E1J zL~9)qy~5_r_;-T)9^~F9ecRXZG`%%l_zRz-Q-uFn>Hk7huXml^jk-C)f1dD10~)9M zHvTq~E$%DJPYY-Gv&PtXbpA&#{eUw`%GvkPtBUmh6xbt>eipSC^#52VQj@3u2j|!| zlt}vzOIHsjhvxb5FvRXWM)nU`rd?njzy;=EbWPJPK*v035Osy-2o;+fR4NZb<0)wu z&4|wEYCW%gaX}jH;m>jR^}3JjV(uoBo|k5jiZtXsF(Hg2$0JAFt(Yu*OLx&8Sa0`V z4rxE278AHj$7D>w;79B?MNY*u^yh{!ogArukR$W=3X1f_`PkhkZFA0^_)bpB?FA7o{UF=%I_R4Eq^ zDGzI+K4?T||3quWXSp{j8>2NxQQN+pxe_&~N87AXAsPc1Szo8Wb7Xy;zR$PT*F|Ul zMP~#@XXGA9Qm(6GG}asYt}zBqmK9q2zrpy#&C;*>UlTtMa8vGD8@=j;_Zn8-(~n`K z|6!T;T;V-WnQy?Zq1ayaWDlZx#VPVMM$Qk?){pijJj2|Jk#h-WtEaH<)CbVVZUAR- z4nsJP3mB|O3BzRHT`3{ytnhcHV+xsyG>k)8y7Tv@zEOvW!e>0Y1E;;eI>!Wh)sXi; ztBps`y;+%lOt|)X|32@}{(rmvz`r=m-&(TkMSXwnU1N-5?RDZNeJZA5I$E=ZlYe+L z%|9SJgmGu8`)1gG0Cnm^^;yE6y9TmxfqyVt*!#aNWJRI$Q7n#@N*|~`C2sbLqo>~( zpPJzrnV5y{+pLEr=U^`8p;dZsTPS^M&$lm@KDFsP(S>e|jIR|a|MHZ7`uiHQ{Nrrt zn`~j8&yNLYkWLqpP0~*8I(a?4S^78?7{3*R$daN{Tecwzg(dVjP@Wu)am=l>KflKT^yD^qVd{i68+6VpShG}Tt1{zbM+TOIVb$9Grgf10NrgSYp898&+n$ocC7 z=J5YqUZUK~(SMmhF~3Vtiox%QM`WMAfl9Lf9%}^552%Uk#HD(&2_y6UgiY>)6V>sI z;j-(*F?28X4d^`W-SF0WpyRHcK$$bd3&|dw!f6~ZCSG2i9?sCK#vA`eFTD@7P1lCQ zs7W#YjRAINaSlWH>+!eq%oi|>Ci&$TuHi?cF#fqZ5c8LqD~$hFj{mFq1=GwgFt&f# zoPyfPlk|`L5BFdFU+(`pbL%Pnzi0IS>f>%lN1^^-bfFt{{HW*W5j3DN)!YKSb-vz+ z|IWD?d?Y;xitvr-86BSaWAB(@d|-<6&I--r9-kier5UeETpJ$eroV&!9dY`!wff5E z=$lr)d2h3Q1EM|ttDiF`PFNi+XZF3TUGFZiejlCeI#4#=nD1N98yCW+B~!wIOyNJ{ z+OU@Usx0BZn42gZ^i|i+75=DU7wuKBtM`hqb6|WZPP#Je=({ZJT{bD~Sur7$T$m7c zCtVY^4onSO#@!wYmu7^`*nVMLczS4DD9XFd+O?_X0_X$ZrVnlCU14>>Eg`q)mXOzX zTUe9+?yz>nyF<0-uJ;~K&A&aY&v=J(&#Z@=aCz7`bVb;d{?3u_+T{9{`B#Rmg;#}b z%dQSZ7p@LZUzixSXWO6Qxp>%pvGK3@-qnBFEuI#QRrf(_{+Ljml&mfxzstEOoU~8q zNo|Jjxwe&`&yoH3cck(or~Ey!Q}oH=)6OE}Y4dN0brj0)>ZAWHzf-2P%2V51JIcJB zAC2_nj4@>m@l6jAOH@=IzwXZsBF= zloy3EQg=YYa_B_cgj-|(O13Y!B6j^~^&ju>0NKpHzb9wN3XRD}-I|H-ledyLaFa=X zhI}8nhZ)I6S=r0|sS==BAo!Up{h1?5TM!~EJrZtgP| z^xUYh#X%EWec(23isoG;jU|Qc+TA-&sfQ_dcPf8($$xh*alTve6|p_s?8UyUD`KT> z7lmi$Sc{W4HMW1Ue#fc$A=w{nQa+cCi&f6MA~te9(e=KgV|uLTzUeWqpE#uHUX5C` zxYmW!corvIzZ9*d(t>~40!2sSd#U?gtB$u%2-F5 zvNlP%-Di(0)M?k(n=9IE{bviejpyw_bI!WZOm)X&&e0<)r`tEO#~c&as>zz;Dd8FR zWhp6PpJ$ZLFs~pzB^+elPnM6j-;Hws8W%wdKaYw9YL!?iO^;|J6cJil@ygPRCYuCkketc8x9gLjM_4=H>xAfd)A(&*LFJ3uK3-tn)0#X zUzn?oUmuIR{vFqEAb(8i7E5H%7osfbs@OhuSMxU=dl!xkd*+QbKV)pUiFwz8$+073 zBU;doTcFD>@e^#F`mWf2C%@`n|1&nFj13!?$)^{O4NowypM72IbKXCH`t7lQB>xF3 z*?*l}4qaV|zeoc)o|{;NApKiMv??Tr*VyM?CjDPwzp>wr4r}j=g>|>3g#7&};i>Rg zhzZXPn=c9%k(Xc$TID5$-+4T2dH9=Q>xWi^cd|Qa4%^c&rG{1++2(JhhHKevVLth1 zd3YmxeU8^h?j}li>ILtPbp76~>L2f+-;GQ>f{G2JL*;KD51*n}J&_zn+_`@-^GW6Q z60&~VN?>HH>FH+6_Ky-e1tIV=B%X1&rcF4Fi^l;l{?nvav+JQ^L$>;emFK=y& z%6j?he(5rCeXQpRX>OZzo}3Zuc~P4AJa_t5T8}9cPG1t%VSBwZF8z!r?o{Qp=<^J97FLD1# z>FVTl%7w*aLaDmaGvvt*D->9rBwnz9v^+oSdQ<)sDWLGzL4j*2x!#wke~7F5yJGI#EYjdaScoyqI=z z=tjb~Rx0BU)hC6ReCZnGO-K!Eic`bdqSUaiFE!*(nG$<9`&oDdU%)p}fXy{yLgAbg z^S@HU)a*>yoTZ$0f8&x=b0?M4 z+_$*Dm2B&q5sV96>N`?G>0_h9%f6*o*}9)z{-IIfa?@)j zV!b>^mwe(zta~^)Xm%!UgDPd>E@*Nk{vAig8p{cz!VC15(5!OR;_RUF^uORJy?wWS zmt6fWDuu7J+v}|KvXjXnnLnvm9g@Q(WS;Sq^6UI#`nnG#2VH%Mm%QJmsJxk~{-qLG zGuaqU#;EWccInEM_n}UI&L_!lU?YCwz88_=-fN*+k$4Cn$3LONx2@{EHMV--)>vX# zI=FCaY~;WD_qlyzw1dQZ;k_Q~kdKXdsUh3F8~UV!g4D33?bg`VSvSQt7vCByJbrO_ z+P)v#vu=uQJMK*1Q#ZzTGVeNbOKb;OykbK$kZo81KF4< z{K@78-v}+IvqJ0e3jIM@;pl~|(0*n`=%9D9?>dzgy3b@u6w5=bg(>ZKeZsXqjY&FUi`_}&Y|UD*W6FWcAr`vN)~-H?8*69Z13si z+CSN$wBVDmXQqEVR+hRv?4P_mlus4^@;@0nSp3OYMcXH1m08QpVgE+B-~aksd<>t& zKSJMYVh#4-863hT$m%|BP4h#5C~_ zRX=?qR6PGgsC@2;u#I^K_MrmL)IAaE$PWAmPChs%))VH$PQLm#+8rN^#iWUMVk`Ed z7VY>heuN_xPlN{ig#I)962C^{Lr;XJPduSh)cJ0CqrxhE=&O;Nul|cQo!Yx}lINDBLz%TH_n_9lgcWW_mS6z7%C!{*15)yy^Q54&DV z)^T6oq<_`@4P+y8(|rA_3*Q*u9vL6c(m$K6?~@$a|G@Pw*SpE>cM1=@fI+;BKjC%6 z?+S@4aVPG@<5-EcSdXpPj&^k8m-sdM@jJYNNz=81a1%a=_1J`3bmP1D9WK6GKMp>Q zFX022$&+tq94ug0Zf5PjCzgJreEqDQg_!IUNj}156qfZ5QB2i=g zy$*p@8b+(hT;{yf=HB$G8k|L}693;mUCbry_qe{7+*e~Cu_kl#JB>w>`uaAhi(PRCd>cQ*FYzb5j`$3586Uu>@ELpoTd^Gn@dNw}zr*VoHB)Ed|zRC?gd+@6-avPGvApZyNi}u8$KNJrG?9bvHhA{YvUqjSPaxO6=s?8Ygx(#45StK&RK&+yN`tjzeVu@GZ0duF(H6}O}HaR|ra zdHy-MXMs2nh3hBuZ;-1ViG{uX%hST3q=LI;%0T8l`kad!nYmxJLtA$nJMm`!WA6DP zy-k@&FV-fl&5ebdyl3R+$oqMx`n{W9dw^8`e zqwpF04*%V^>N4-mudXk^nz|=+S@dAIn_GtX>tFqx3jURVd`qZXRd^3>R%vAR?}r9t|8K?XQ~c3*EFR$A^(ajY ze#^JXOQLW5S`?mRjagrYo4MCc>(^lIpUBPBH{3+eUL`E|WV+_PiuG@;a$i^U{jXx* z8-DH?_8miSxY-!}6XEf6OSA|&3_hFcf`W)`5*1w zoy$$vJ?5lb_Qt!eb8XMUpuQ5q&U*Qo>zn-d;#vNY@7#ObJ8p=DEm!Hwls~@0pFJno zW2X*~9sJ+**?1`S&8y^%={FnK!(-Th{^)tv;Unf<9`MBzU;L79vGy7X}6 zHOgN4HRL3`Lcf8$3AbXfH5P6s?~L?>_ZN1rbJK9f`_r3*S>rMNniuo|kZp_fhfX%O zPe1BfCpR73b_08P> z@?LD;@w4zir297yME?gLz)>4ye31Mw_B_@bJ{s9`_i^%5_#pkm_$WSw&){?T0=|T= z;_LVZ9>+?oM&XV2>>$@;6F$SQ&tWS)%9FN}yYMHxg4gkq!L7b$|1B2huK$sLjg2BN z#-+F%W%`mnsjuK5*^Mgfx=Y+42lqc0+eP+Yu{PGX`Q=#OhLTwS=0A~D%+0a>@X>gG z;+fdxo-@FY!Nw292Rj;KgEy{^eT8|ns^8#^+hc1}ei|;JOL~c`Asa}%3%B7tkPRj7 z$3yrV$RZLS!(I9{XQD6hm}gxZ>%ZckVttog9_!n8X-w8{-r^;(!P_2)y+3mEk$Yo< zcikK7zwi2(s#s$1=1H-^vG0ie9sL^bHwmAh|2=M?--PA#Ral3Oki{f+;t6~nOYv#% zaVxjCLw1;uO(gE5zZV_!Z(;}j0bB4D?8i^>A9xXodDgaJ6;>m6nK>ER#^16o3Tu~M z6xPkR#!da?sT%E>QvDAoz(&X-6YusO|3YqYw&zyod~Q>I7a0_Odf57FatCwq=}}?l z)GPGEjtaY(OO(lb3Y7I3SA^mz`Y5mqJ5hq&xY9T5Aq$UP6gCfC6tnm?jWtX4kucY$>mxxaeV=@-%=KrCbCegVXRBAvMtuOUBouY78*vgTv* zhb7YeKTC^`ORF2@3HzkiO7GlZoaVT>BR@2U?q|l0pVMc^?&G!M_5tZ%n%$`HcW;z- z@qT?_7vZDc_ZQk}7faup?vdXxU(dXqEMs@&4C%vtPqBNQ``c%_&U^%A%*TB54_*6v zWYVYMW%pg{dZ7tTW&Hf6YyT#E>U>KBn$h>u2jYF7xH{hd?Co*qf5rQ*_>XwsgFlJ) zKlpomJKo0400tj>AwKx&|A-sLO}r49ANo~%aL1H*U*r0C-+kNT{ZBj-@83bcV^-+< zNOin_;!16z_lG}lI~2LU=W%uBS>ccLKSkz~PsInXTN_U_T@`XpTxI<4;;?$^#l}6X zfyNrF9lk28>opI)`0DV~*=xf3sp(C=`@-4wJD+=L*h+3Q zZd*h?&Agr5k$q_>CU-LL;?M2{#&xl0&ZS{5_O)FaO7YBzOG6p6qOM_s-3|2TpMG%mCLAI(dQ|DrX?`hOgqVEsQjhRlUT z7rOBmVKIy#nO=v!WbvK(C%A<9E_@JQ#7aDkUh(`taVfX&5XT=HEzaQq`g^CCKZR2K zD_U-nKjB~Tef$tp)#1L3ZPbop0 z(=EC_F4p#0e>C+fX)7)4 z;O8!KCraj|h27-dWogI;&yxpAH$<~7@x_i>P(T;z^b9fz>YUfPATx`Zb zq>JaD!e{U~{DHgM*x!fuV>0Gp5uU(TunphBcd;L*@Jk%QJEf!1I7`2Qe2M%ErqS=m z!}vRV8ec;;a-W4+gvn2h&e2Ik@0@`QhvC*-B*XTs$dxrUEGluP7d z7y7x=d{2z$_Ga9Rzr`XvinZ8@z>iCDEvDi={Nrf#GpLp%ULPgTO6A`@mxsH_sqF8- zUAWP;K4s94j2Zn(ne#33Z-<%Kqo&Sm3`yYIV{-4YBx!~s*ou&_jT{S-c4sHE3?OfbN{{*-n{3*VN>n;y3^26~wiE;e9 z0zaq!6gSdGx%N+_sS$}4ilF>0gr~^$%p2GjxW19x#JoA{flz2Kfi2`# z=56eYCOib& z$io`?TC7{)|Kln8dTem502}F>uz806kCF0!?nTbo(MDRR{3o|F@5oaAD|dI2yO?*g zFL8YjxtDoglk&e)`A?ShDF07e^U`}!h}KXYWUe@G?aQ$8pR8i8W?$p_VX~IFuF3i) z_m8YEpKE>7e0`$}&G#o;m|L^7v9is9Cfk`i*mt_#MRqe+`*$_v!m~zrk`>IAi-q@6 z;Z5%MughoqA9MW=avyVPivQ(bmysPM!n0g>y1$!jWp2w8-UY&&Z0z+v*f+c0Le??Y zANRj{{4a9DqM4xp8?g!NX3Y%wcna&0yJ%*}!y2qbB57vGf%%$YHFht&CzK3`Pv^uZ zatCwq?0eL=?@=eX$9$A~Leb)T!qZFd3ERod?kU6;Y{j?bLBE9QOo^AFD>^Yne)$BM zi;vUykw3(@@Ev*Ev&yE&l|?^h@an2o>1$MIP#K@s*uX770YrOHiwPV3)Sp(7XAKy&AIT$dVf~!ftJZ z66`szU%4kKJbQ<4)2{o1G4{R2(|?Ie`E`T#;p58JQT%Mc2O~GGeK@lB+kWR6TRdYE zH$QWImuo*@U#yH?$-JI<5`Ql9Zr|qj@0iDW))?+5xqbuxZs*s#wE_MW@8Q>D+^%)~ zC$9g=^{wuy!jJg#CvIyr%&R=(ySP(3_UpKn-<9K%!Uwrq%}t&D_|Q}}IYzG@(}O#EGF4bK@r>yb2&-=Y zKy$R7#&Is8g^zxR4h3;xdi?{iN=8zTp+rrxgn)BZW9{bOC=8mv`;T4$0^enu=j zwbc42>j*c{3#^&gXnn*ca&xw^K5`55R&raqF#z&u=I!JT_Y{*mnRk)9T`wW`Fz+Sz z4O^MSXit!PPMFIvU`_{iVJGsA zTbqxySc9B#mq@JFh1KJ&>$g_FfZYac!bao|m^Y2}cnVFO=Io$F+G<99=XIe@KUPDH zIX&ncHfIOju63b}-ir2f=JcT4T}iAUY8zknOLxq;n7 zWP`XA=f@g+AD?nP?*3QsVfTDaT%5_TKVYKgRJ&)jxcq@AuDiBc+WZ*?@dwwc-2bxs zo)=%g!~Jvk8vC!iryF1OyshqiFaLL;pWCl{my7we7Ju*#S9?~>yVr96ifg0!@fhyo zPYeHl?fzT&b2+yO>vnU7ZVsy!YyWF2=aP9#wg1VrCEEYx+W$4$|EF)!|Em3u0&K)4 zZ0^(k#};hGHWcA$Y{!mL;~&_GUD%Bh?7?2_TV(u$|Ic`4nP=|z4&~$l=7VI#LT&VN z|7zCQP^I7Uf88JD?fp&s%l~q|8oSmr;tYs;aU0b%kv zd|kM1#g~NV3wQjPvY)Z z(SIf1;lJD>{O`t<+rwo0|F7Ni58Qv0f4?DrL~hr3ds<(55xGsj z_||-Li*n2{IyW&4vd;C}5?bBhqoG0JVaF|5{*T$O5$ zQIg66x#^s~dK6#-)~A`9glLY^I{^T@bCY zK8iN9;^34~;Q-1tO828IWmI?urSyFm+3$FQ`Qqcu7bmN{*CDd9?Z!~y`qzcgCgJm4 zVbdV&e;NH};di^R`GK(ef&VgyN!%S|_a)(Yv2efBJ%1s8k3S3ltAzh6+|8h0%-!#q z2f1zF?-l;pKXNxyT-%6Wxc1+fZ{p_*_^|uFgs);X;@;t9a=T}IiXWHq<6eB6{ipC5 zd>ePN`w{Nu*C+86+~U99jpew-v)+%t#Z!11E4<%k@6(EJAu&t($11EwF7mJjYq9Rc zts#H5^ndDB^?&Jqf%K1`yEoT&Y@VjijogwM55FNlBkYQ%O8--&|6b{TK>9x?{gb~MmawZpr8J6;qzW$aVW+9pxjoIHEG5_$=YoF ze`G!L5wgMcMzV>ynQY0_|3|hlA0^ve?--bBU8AuTbmLR}{y08^&*5>bL>XG}6FeXu zd;p)qYWxJF{F|@hO8@6}d=6hg6>2fcf9c0f{?h|kC%o=@(f@XDi#+3B{Hs;b|H41v zX}rrn`)_!s|MV{WH|)V#oQGxHiOIN(UsvH_d=WcPhka)&X^Q-SR=p7p87k<66W3HUi<9tLE(&>!e(|`t;yelBJ1+EVSCO5ZTv~j zBe^!LJ8cc~xogc4zBUxhn-n%IyeVuVH!heIau!}|py%4KdWyB+8P|rqG<)gO*Pg#N ztg%ME&D#Ig1?i!kJc>?h^g9O9jZIIohlREKXPhx`%G!PJaD;4P*XUXcb2It>(Dv@( zQC;=^|C?PYCnaMqwYf_q|NoWbAl+u7G(PG6GEh=9{h5lY^(&O{|p6C4gduCqiv-a9+uf0BN z?aR7vx9bMAl<{o>q{q$L0m!z%4SLt;$F<{Xyg{{44|UK84bc4P5;Y-h-s@%WC2kSo zh)ibNfj&<3N{ zc>jS1dn>e+as3mHuZZiPV>%9eMxB&jyNF*mVo%{ji>E|GRJuKF9U` z1-KjTfrsIFcn?0uJrF#7MfW40he7yH^X!5GurZdyKDk7R$5|iYJm1h_+=m04=NNBt za0%@%Tw{|``uGxM;3xC=5@j(CBzr6MX`1aWOjl#94F#k?E(W}yU1FEE}7 zUVD#v6Z{>1O1D}FNfTm8{RdDP-X=8w2!&#Vu-@sci z_cPRs;XUlXhd;svn0n!U>mSI!!DaX>T!lZXwYF2r z+Qayt1GDsL?9Tm+|J}>@-v{aA$4@#k!^`;JUdDn(82^jRVLSIYV+6R)@}U5G;W%SK z4|DyEaQz+R`n$~acZsnDxL4p_iN7lRMaQX4asM-hz_paVc`$2@G;E|!vzfX~0%e<6 z>M9GUv&^OZyO=u563V}`=;NP5T?TjaeW2*~h4)crg6{p4q4rRg+Dlz(H+8CAl#w2# zthAFd(<;hNF_fVQ^WW<~ajtdfx)wU7@3%V7-fw*w`<<``w!qim8Pf6u`SOK@j1hp( zz_qXh>H^eTBh+J=59I2l-s+{^>ZTs+y5DMSyx;Qd;kmV!=U5T-*gVF!ohIH>=&_II z*na8_d#E?>r5+7=*o&#pmK>lSy^p#nPudTlax2X5(`8M8|rI5n1eObOK67g29tNIiZr^?2&S^`vPXY3o`@ zUM!kPbCdRsq)o$S(%huI$pdJ?-cI_PZ?kyu+r}~72T1#uD6c{X_I{pKW^Hw|#$pfY zA38g;Hcj9>}!QK>|XF~ zBrl+28)*++JIM>^!QKmI9qfMi|9l^V{bBY+U|=J43ph)f{pY8to5ONg3%A4F@C|ql z%AgTO;1C>vxz}GFd>@{MRLFuND2Ezo05|yHAiN5{fq%h&{0C(`_!8U(_rhMt1rPK=2!`MtxB|9u z%Kwl!Mfo3+u{*$diSj?BVo!r~_GLgO_AL1SJ^xuOr};AZzk+oXH?judVe)^3{GY?x zgHz=HtOeYQ^sRT2|IkI8qVvCwQ~o(^#zv9ru1B##q+|Uj0ai4ww|9~gC z#-4?r!YMcd3pk$Tlx#P{2DqENxQg&T0iTBwsDvK)9btS3zu>q}> zU^DLD!o3LS?z4TI>)~3s0g|}ZQehNchrhtz;k)=tWPnkU8Dzu&D?8FT!gmr6&r$N! z966BDad(t6QeD*9*_WL#GtVeK!2Prt9H%W|7vt`x=Be-!Z3_Ds zcXyHb&xdI%V9Z1r<&J9h)f}9sN@Nw(ZDVW&)UKe;j%>0sPb!Uh@MUwD2TENXdn>en z8+=(GtuKT0I?TLx$~Cx0*Qa5-4|_lKKraOG7ozjPk3GQt0k#MCGY=jaVcX`IEqm5% zo`18IR5V-3f!T6^(>+@$kXkldY16Zn4jGUMSy^Um-)zeNvnl`2R-Sh@^M7Wmpm#R& ze`e2&e_?E2$qxFz_R;^<_`mdj(Pve2?rPOCro|P@`UflM{~{g@jNxx2PE8!!JjHyn zz4U+4cjciEt93E+&FE0^?qw*J&AaLl9ntKfnd}JG@AEo#26`9P>$71oy&LxE}_A&+yn@LhgfdaKT#K?t-Ub zKm0dTK^FwzSMWZZh1uw??0YXE-&vtWPOD9AF`3{CS>zzo`3shN87ADyt7T4bsKoIc>Xzf|6S(&w{W&P zxhA@n%$8YWvxn=!e0M7Ped)72{|F;MSV6AYkc0fk9&ydmbLf5m(qIRC7mA<~9)T@9 zQ@#r4uzvtY;4N5;+oz!k9w#r0R$ZY^b44W^uc#QAx8RC$p@8lDSy$+ryh5AL70Mk~ zXh*!FvN=~&ihBjJ92ys2q0i$AbKS3~Y2g*Q*jI~NJz><%y+T{h6|VU!jAy+p}JsMi2CJTpzM?&lSE2bVc2}uc!+mo3AKLTm}he00K*|&?j<5A!Lwq z+Rl9_J0xEEP)U${_Cq<&e<%G$KQf2yX40S* z@*k!A^C0D)?UaA^Q~r60@(=QVE&t%BdMo))J+YQK4F7BW-%k5Cw168t&0L~9mt^jlX?L8 zbBN)ddli!5OWaczxp!{nI>;dZe*s=-XPY(DZGS_)jGrL(G3@p5N9^73ZNjRCAK>E->2Dt8 zo!g6av+afM|3kdscQix)d@k=fxDEG4SPSdnb712hwnKFNw>{kd$1lr~c3IBd-2YXx zbOU~t!5830xCMSin7;vv`Zne_+Ukf4(|T-Ya685}#eW-*ciREvW#Ug2-j)J+V4-IP z@{`C$bfLZYxciLlzXlB&;D`--Ud54elh5UWU>l z-f#9!#gj+>1rzK$1}|cN8<{)xG37x%&!z$>bmv<|Bu+7zRpMohfBYOl?7iC&@KdF>=78C{4sdI&-$O8>fFrx zF@g8v0-le&1ADi2F+Q|Q{fkjN5RjdBZXzT>GW>~fonIi&2qzUkX_q>bez8Lt7dq&} z=wSXYeHv4ggDCsta-KZMXMYlLElex0is%<9E}|R+r2#Ylpj+jqyH$}!A4(a0DNv2Q zW((tA4)Xqox~1K!A7T7U9?ySh+R~*b2(RTl@BfRu|1b0YM}7g_Z-ivhChzr z=prq1+3pE22Lahf`kFPC{Y9h?euBu*Ueaej>GKllgB)Z#F|XcA0$Ua3zp8r68K}2X zvg)m5aP-z&=|%Nc2K!Sx>#el1dMlgybdHyKQ0&j5PMuFVu>f)>=TQD*o;rigOD-~R zePoV`7+777H0%47j?ba|$2@oZ)|_VEJDBzT>SE@oe$^b+E@fUkG(qzU%6|!_{KvXS zizxpsq5QX)@*nsXQ2yJ(I!Lo9|3TMA);-!x`42idrjKL#39pB6dLej$e(#I)gA-2R zGX36X>Gz(dAN(Bs;E=!MxC&MrS2}6-JDwNCOOLAr3O65DQNnT7Z8@$An0lwyI>+_+ z`pXX1-f&oz`*8bZt(C|mgCy?>{gGq;FF4&NC^w!^GB_Z8*KuXQRC29#p5wEz=RhXM zXTb&BX8nQts`f2?4B5AUOA>wo6Z5&Akl&^~vb^Ul#t=|1fD^c%gh+3S6&*LQu)wO_ zNBavjon~Fr&+*)(?W2(gk%tON>)Cl~;XUJ?^$Gd*(KgAkZM`$!S_Fmre zQ>>YPe7@SCzHGkgs^+t1!hF?YZ_Z+DN!onM$n(_zA;Jyf9w_2l49LN$RO=j3Z6ck* zoU1!9pS2<8Q%9Vyj&aHfjq|A+&R4H{KJ#eMYkZ#miT|Vo)<9!Cv}5blN`{m?##gi6 zSX$8_Z6Jfnun#JowPLakT+OvVPdU8{lx+_ykB?>Nc7}PjRayCYRaTKVOdTMslH-Fa zhO&`Cl|n^ljaBZZ&f^|X<$lIfFu$St0OQMN4XTzk$m)waRp*GP(H^1h6XD$-QIorq zI!{C`xVPfw!S1~fQCn6-?dKx$U5u!6>7Y8QBI@2Y&NCpQURQ+i6cP0!`?hr|uxFe( zfSn3)j^N==MaHkD58!GIE}o}>y(H4*0ojjNTZy{|lr%D+6fDtMK@Mitt}NJ;38zEvI=;Q7b;Z;bhR{0v6n#Ur8j8*eS`MjH&ik5hAO8| zsOsFzSelw~^q-vK`F~1vP=ETA8b(g3@$e}%A)7CsqW$9(;}K7BkDQ|YPx0yTpEA{gcWywtkaxj0sD~=}F30kTRof$Q6=f@PNoE7LpN-F~ z*gEI&PJ@29pD-SQc|0dS1&g3_jPWP4Eq(A+#z4JK`&FT(&*I*^fHe}o64nP_po?JMtUH4a}^(h|ymWvuy@#r+Q{+#`9*srzA1wX@!H z6>Gsk)>(7^U#*<;w1sdlWSC zs0P>ZS*(pq-4E)Zad(#*E>f?!Fw+L=o}6g|ZAF?kP;cx^8)!Syw1Ez+VvTd!GrK9< zbwMv>y&mX)i8fF$Wj{YGNADo}SsPbnKl!Crv0d@XD+FANQw3Elu z&vhLzfHQ+Mx*5Nrh0+I~q5b?;-v7V1v=+B4)~?U*Asv50`X|$dPns70mHcS2w1NGb z;R7ZV-i^E+?u9)3eb7dpFsbk%v!DFiVcZ|L^ljw#jGf7(4YLUcng1QaXa4+d&=3EYtzR6NXi3-LcrfA(Jb9PV7A zcJ>v&Oq`J2WMc8(EsbD*1OC9h{%hp>a0#yCUaFx#_yz1AyuU=1Mz$xZOoymN@NoWy=6>tsMxIg2NZ(U6ppJ&r=xc|?> z2dPW+9`cWH0sadAfXkpy^4x%r0j~wyeE1A}8m@<0#tXP+(f1Gaj8|xwbAuQfXj^Xf zExBIB_GXnVXw+vZhkYG4mW8q5H3` zv8ai1Ym>5GYEt%|CgtpIQm(5(c`KS`-har#rA;b2=%FpBiF$xXrRRA6ZS|-;;It|T zr!vr_s*6pkzR*NFaFc2mHpw+N&8pkhMEhsD)xh~0d(*9^jZL%zd(^ViX}O7?=TWa( z8`G_}1u2#{wn;wxwa;l{Y)HD*c`)7TLUu1{QqK<3@pO~=Mm+SzHPQaxq`;(ydO(vx zhnp0hYElIEf$=5{jx^~A={n6aznOz;XXLLA=P@l z0sI-P51wH?fZHR+Jr%dgREyWK?R#c>cAEA2aGLcv_8$Vv5!s%F7vO_4j6NNw9)@#vyyhz((Yd? zM^UYud9_M$)he~KR%z}WEB#z8?O&Nzrn^>Ii!!b3vRdV=YGZDp&B{ApOZ#^#^M5m~ z!t-{kXtGwt_%AtIOZ#`GRkkbBDo0jCY8g*oCtf+W>fLRs*;mVYakL>o9p|e=-m#o^ z5zg7fc_vA}*JuxL6TatCE%o18we6_WHMsdUzNz-(we@*w*1*PE4PMT)Y-KL=&$1HTE+sAYa+6Z_iMZrE z?@`L4EGxCjW~I4YO2=PDkxQ9Nv#9rGS=q>(^DgBctXJN4m+~LUNs?`6TE6! zbzE+TSDpiQtM#BuZ3kTPCb;Ayoc38RzAxjV|1;a_n(I>c;Vi3%^YvcLvik7XKkk)( zvE2#~zu*!NebHG~c)QJt>~m=Ve}h|z&!t4G%3G)E%Wl=o$+k+<>X@&UO?kbJ`AR+& zcGhW(G$}q;r;?a#EB8_zb)Guq<6dB|qd)L1)(V4Il?s_~hS5eqF3na53d|WY2UR-x;V= zALs37yQ8d5oq2WYBA(rjI=)v_N1e#WeDFH8=y$49{Z`_$Fvqgb%CQoWNvG?n|G4B>Qm=2IbIQVc#tqafZDG68k>AAa zQOLqQdu~1B|LeJy$^QlQ%3oZsf<^VDmrK)Jx5ZN@RFY7yQv8-(XjeJ1Vx*n)KA{CG zxc=QqRt;{o+sJ>ysUw{F*m^Y}8y9CN-%b?)~+wos(nrPUcvB$bRDG-{n%^e7(-BdWw=q7dO|3<3YP--TFX8(*U;EiM z+V_04e{uhBdt2Se9#@-sx74Z+*}tFrf6vlW&Iy7>mPVkv(-!cpI5HK zUdBIosRMY`u&-T>7rFlzPpKK%GU=5&)+Gf4ddre4$G&h395j5#l$%J9Y4)-73xVF_zoTI@LZ^ zEJ(B}(|puX60PchPc?ISR=ZECD~9%|OWCY*&GX=rPer{x6?gg=1M1Vn&9tR&V0`iT zNz%v1^U7y?x+j4F9vA#0pqO*xs!}c7jai333#H)F$kC#EVWw*C0k$Wqtx1I7& zlI2*KWI2&3huf9v@+vKc`=5I`V`)43wJU2D_dhacNjvM`w<~W~yYdgTt6+aSV{yoT z+>7V7GykVurFQZkS$^&meO}!Ev)Wa~y;qH_InMo0ecjc_{l7EEs-JFG!%p&lQ9FHM z?P^}yu9nz#xmVd3N7b%YZ<5uvs-65#vV4o%S)-s{9UF73&b{sG+T6f=!X&F_Z?@ID zqn-6H+SR|io$^nT6_`%4g2>S3cGB0Y$Yt*Tv-PZh(N0H#hw*PMw0-?jNyy|~Eppf! zTO~D=N9GITWI%fQQmlpm7mt4g6+vx;dl%E zU&&T6{z|qru>N(bRd%>Vvm>o&^F{cCCe-b~vdZk}DOT>rEQE}?A@>BCQZf8xRN*e123|CI*DYKtLN~b)yue;z9PTX zkMwhVUoJTb=POJ2iM`gW~=W(yFdz8G#BL{w+2PiM?_t0_7^|Xun1TyI*k21MNvgUYL zFa1}%FFm|ZJJ4Jo4bLbt}gaZZk9-&#)TLXIOP-Ic_?`stvg10^j8ft9>NHYCG+bcPhj3@|X;(Vo`=wx`2F~n_;DKs*mw}uRuPs zhITV#^-c6KAZezwoq-0P)4xOh8Ta>*ESF;Y6Y_?2^gr!nJ==w`Tz|3ZrVXNJL9BXT zVhrR~*6?H7&;Gz!#z4-ERS3ekN0!8DU{Nf6^0B%|I0d{vK1G<_|nt1Bj*w=9!pB2?T_)Vc+aSuEQdms;< zhbHd(=2>y9JH1AYff3bpuA$F?Ho0AES;pI11^n@3p!bd#&!OUdy>+4fPEAomZ`q zV-{^u=hi3*Z0Fa=j(aw4ISbd!j5}DeM(JS2A*4duqBYdBXw#!zun<|4w?_GnH7dwj zqui-D<&Dtxgj)^mNR_8)dt_hvKH4IS*3chE+a&u>5y!y7m3s6&%7?2~>UDp_I{ckM zYbAO8CFq#Dl76+7%&%Fg+u80pyp;I`OPOD=RQ*{?^)=kwOIA`pS*ccN+p|($@VS;U zKWC-B&VJYBrL=D^)i<#>Y+R{~)r{%ke9ec&E zs~X5nTSx7z3ZinuUN|Sbt!XCmeOvsl=9h1#;302{Y1Esd2Xe$AbaUjPHw3PP!rFxDyI07r>y#J2GFOf}=gIkUoPJE$`S5*o%R$GVV}A;}57`IT zKpm zc>g)Af`G#+++nkdcwZEc#H-|Tyh_~#DmxrcKXAM%PRFYfS+&<`RUf3y5Nfx?%LR2% z4-L?`vp`MAW{1;iiO~MDFJ7Lr)PIp}TjS;39xq>BCH0>Kb?n@r&Q%Vpi+igZdT7V! zol2m-NI!6l!}8C`<@w{Zf|n8$n$=;2&nECbN#K1loB3a)-z+=dze=$Z7sV@yaFgj{ zc1$O8UluE6XFTt#IZE3bkN$Qm<8-o>8Jl8d9Za^eH`=WnWbWw%ASJzoXL!pjNt zeajPRwpoqa64bOe zLCw1psQ*;S?M46CjcVPJY_*jY%L~3vyVc$qua4>2+@Hx-S4+ zVQj{^6sveqqE&Vr9abb;70B`!+6wY$?{%+J4YK-Tp6ncF+n#781xQ22fjV|1Tgm8^ z(%WS@*_U=;y;AqBr|l%gN@u)5)&bIRKWzmY*DHJ1dUZa!P95V(R`$5@_2TyllL5H$aXvU$mc``3}tg46hpbWAGlvr0?Spwd&I|fA7m_C zCI{|m`Ie*8NYIPp2RM+ub zbswyx{i9O7)4BA2Rx-D^QvUHu1+WLFDyd&qD%@E~zeS}6+?5(^tYj>}YW4T7R?o%N z>h-Rs9u=c*_iA;-#xSomM$d5$CnQ2S1Y=?pS{Nh0eKq5bVpykewVKbZRyx;5l#@tU)gH4Z;hc|8Y3?Rac|ucqn2GUa<7OXomSIMv|5GxSCek5X-AEr z{I*)ToiWPGiqR|he+w=_4(BNM#;77MhB1_@RpyS7lVel%tX8r;hI(6!5)Z6a(k>qM z$75vQx?0(bSJO`)qs++|WkDffWE_l9>U0e4I$Vs?mDK;LssH_f`d>BmziPtCmlK)N zn@|1k57hti>HGXLczwu^<1U{db-^rhd!%AM@0;V4k{>Jxk^>CU&0s zPX9~&OMWju;RFaThzt>a2pzshyq|>^O#HWBqr3-Mzw=Sn??e`|T{MgO>jBp9gwk!S z-wEYASikeZAyu-!>frmTX6=L;FdhGPNZj^GC2eIb5TqmTpq$uK_OaFnoo#7zSnI=e zP#LpW3j}-C0_rur2bGh>_gm8DsS~;!#NST*p*M^8v)#`+pnmoRs)+x2;(nIz_f5}Z z{03uJrucr}ang@CyLPgED0TJvRjePnk?%+D9Af>fgRGx*P%Zn0WY&1}EMbkuMTgWj zmo*+^4$3!&@gtY!aUt>Dx0e|EL!W+Df<2= zUuOQx)yyMf{cpyi<<5FpdHeYO9~3|#;{uBI&u08DHi&a#HVW!5r4HnH7|Y+;O{o4FAl=0dctVt(jG=7)k0 zd;5d0spCQBchV@`h3w}2+q0kfol}2QANKy;%n!ZzN9O-AKNSBV2y<)%2Cxr;ZE804 z%-PgCW-AGjDHl1wi9H2U*_Q_CjLFMjj9n%&D~7QTjjt(ZDf%xsPx~+W??HbkJb?Z$ zF%ED)eZG4c_xIo&l|9NhKx9P~f`&_^Z~*cybf=|@8NxzP5pB|TnB65 zD97G}{5))cufPLv9QW@a?}M+xDCc<``4qeWKZMtC%S4{UuOImjcneP8ehiL49d&~y zxPtrdpqppYrW+|g#4&Ccu7|&H{3-l?3H#0PFkHs%G34#=Bs>GpLqRX|`XP@#|6C|J z&bR_7!d?g!=UEp3$`1ZorCG;SgS{H6pc0Zc-@y1l=J|sSQcm9>CpaJ(GA`Yq^b5@4 zMyBqd4QTVPl+8T;tSRQwGZ((Qk#Q@~Swvn!KXViM0_0)ulzMu}vuW~<{0@*;e&itA z1DD9d3*;rTo^=4~ihdEo=y~sEk zc#$+n0*hyQ2J*M0TM6>xY*!%PWxE;K4MQ*=_j=NDA@a9u{|E9<$iKn6a24goYvFV7 z1-Q}J@pBFG4!9fMBg}`9pT~U@@~-e<7?pkbgsd8~I&$5^N7re>}iApnX%c&r^TgPW^E| z^+)E(r65z;PQyK&xpEm(rlO`48jV|IjnVwF77wp-|y9(L2hj$racd*@!?Ape=ig#NW2EF7N1aSAW zKeU~9S{7p^Ij({2W@Hn`yVzgLc0IC=Q_$|lX zh`bH%fLq{h_%>{UZ@~TVF#I0Be}?mLH(|es+s_~gK6ajG0`hO{yN7+>fPZ11h5P;3 zo7vY6-7p9t7=C7Zebcm7&sv$+4E0xF>js-b4flxiV!IjHLKtqOhwWBm8~eRTAKUH7j+dDKgzRFw8`(39@xR2ik2v>( z9~}Y^#2)evE4+yLPlPdm9ORhDan^)~!F?kd*gv8Gc0Yt52)#wDDG&Y72c6IX-OvSQ zE&4WS2Oqeh1zN!ajf*IgLNhdh3u>Vr>Y!@s8p?!}rH_xO8d=76DY9amGB>iQj52v1 zW$|;YsgKM%H=^9>5f!9SX7Bw_S!bD(4>^zxX(P<~uzm2>f`3u}0Lp}p9nr%qJ?R)IE@vg8# z;yL>NAo&vWKf#GT1yb3U2I-WwGah99neC$gld-0Zeau1T9%lWSgip7PoR_1%4{oS4E+-oj8Obg?MDZAoktFCkOkFkk7yiP{8(w?7NCF zY4hM(_&M%3a}VDFYk=Xpii5Qf4+*di*24za2w#Fta4T$vFT-u{GMF+j-^|kx3!yn;K@Bv(aKf_<(uW%7e`}yC%jQu4m#5c(G zPxu#HhAZ$P*uV~nkOax#04JnCDx^U=Fwb9^kOkS01G$g~`A`6bPz1$L0;Ny}f^O)6Ug(2<@IwHC5P~p7U;w_& zwWZJN^RL?>36jA9PDp`N!|V9LG=5SKCt{{U1{4^gdkdm_3!-}qaV?Cx7DinQqpr=k zwm`Q&Z?+F}mMOxR$}~*nVP+Ykwrp(KQCkkSoTx1qTW-{5{N_z{VCL)dzfCMNM!A-1 zDgWE_f3!@?wL&ZP1$|LpWR?Ff#;!3&)D~xKYmE_aj09t>Gsb#jY%s<~4C5`%9A&&3 zpT?u{XS^9-#?uzK9qxcTVJqAPcLN_4_~JJB8r%c-!glyNd;{)-9l(bNzIZ=801v`L z@Gv|AJK=51xbP;RV&77#28#VKyWV`iDXXS0m|v$qoAbWcVdD9IVw*9Gl8sr~ z@P6$_mh%-+OSMU>_v_73?{}COz8^4gJD+4iKA&mQ=X??2uhMGWsGAg{n@vB!&2S43 z{WUjf&COc#8+-Cvt$8;wz&jxJCdJ;Y*fso(ASH}hnTi?pfDf}jHo7A=x+6BaBTljO zAjHN+cf>_^#6@?kRqR@Jtc~ti8+D3TY&=f!QK$H*Q-Wd>a7u_eB}ARpDVDzX*mY5- zby27Fid~P>`l!?TsM7|;Zop|n)M-Q1X`^B{;RB2-p8BETP z$tf{83nl?$5>_;rs6r;FIK|y$`by%~@HfKjj*BKvT&&`vzT=`v5{K7la>PXwB5tkX zqKOa}O@z33#YGb!E}8&w35ts6q|-2@qp zX}scZ68S~rHDe~?;}pME@i->bBeT6>ZU$2esulsYQEr zyKdj7J9NjLy5pIfbq8)wKA}5x=ht-Sw{+)Eb>~O~2DDXo>8>r0=x*J8yYBw#Pj&Zw zyL9&feN|t*PhWjVUwuM2ZqqjNhp%bdgWC3xwmqV+=^ov4yY9iE2iRwO!k9*Y-QK{Z4HsfbE33{hQkUkhVXp?GtW&U0?s2zWyV9{TY2j_vyZ`>c0E7 z!$Z37o4W6Tt>Sf7f$* zOpo324D8oqyPncxY(B2X_VSN|wR&8S->%1Z>G7xZ_+I{Tfa_27IElGiyC2f-=bq3H z^aJzX15e2a%)NR-PY~n}^+R;~;V$jb9`lDEY0uNz^JDG#iGHLf_2d>kiGEMw?a7Dq zM1}l&5_2#AI9RJEU(l1Jdm{k)#h zGY{!mJ!=x~*)4kZEv2OgtdC@zp4aow>G>D*g7)jj`tctA zQTFIX{X{?6!+(zs=%@Ob{#!5Ul{&pruU8uNN|Rn`(<>c!z+FP8A)AKm8cNhql7^Bs zs7*s&4f!owe<;YJNN zX}DR#EgE)f*rVZA4Yz67t6`sp+cn&w;Z6vaHBzIIT8+5Gh~trZjWlSaQH*RIacjh*kyeegX~e4$pGMj>(xH(~jdW?GTO&Oh z>D5S|M*214*GNDkL5+ko5*CjtZmw6e^=htOt<|e8z1pf*+x2RPMr|6kYcx@#SsKmO zXpTm6HJYc5<2 z(H4!mHR{o5t47;2>eZ-EqwN~yTs<1?)u>;i0gVPV8q#Q3qY;e`Xmn6xHjUXemZ-5L zjU{W$p)se%QZ$yTu{4dPYb--!nHtN|ShmJ;G?uHeJdNdRtUzOh8Y|LRvBpX?R;sZw zjg@PxLSxk$tI-&9PMS2(E%I#=11tt+9TM=W0Ap z6jW>!hqT|hCH1fDx z;~tH-YP?P3UXA-S-mdWujdyCiOXJ-d@6mX##``qR8T}d$XgsL#kj5h#AJ8G24%u}m zQHPRrC|QS6btp}T(sd|9hcb02ONX*`C`X5Kbtq4V@^z?4he~y*Oou9Ts8WZjbf`v$ zTsl;zL-jh;phJy1)TBerI@F>=ZXNRIP^%6FbSV6^Udz>Md7_p@6)RddqcS$()I_Q# z(ln8-iA+soYa&Mzxthq+M7}2QP^gI_O%!XQL=&Z&DAPo_CMq;hsfj90RBNI}6SbOf zX`)^e4Vq}wM3W|(HPIrTx)UBvv}&SF6JAX)@V{LX9h&IWM3*MIHPNGqUQP6AqF)n! zO$0O%)I>-VVg6g-E_etYh9}@D*bC2VBBF@_O$_RYO-JlHlBgp|I+Cm-4jpmoNQ#c6 z>PVW7r0YnAj%4acmX2iWNTrTc>8MRd?K+yIqbWL?siRpsnysT{I@+P5ojTg3qun~% zqoch#+NY!aI_lTafQ|-rG^C?p9gXPdfQ}Ao(xyqfCKEN8q{(DWIyC9jWQrzJHJPr- z3{7TgGE0-$n#|Q?o+b-4S)|EgO_pe~R18X)tkPt)CTld=r^$Z3k*7CW^+vni=+rU0 zjwR|?j*jK(Se}mM>sWz~73&zoM9XxnT*oSOtWw9SbgWv(YILkl$Le*gLB|?(tVze3 zbX=W*+I6f$#|XSj$GUZ_N5^_~%&%hs9dCRqQOA3ZHyWeW7+z!a z>G%M~iCq3~^&7))j4;NDN}b5#Z#MGY8q|q={$`_qzmbLfjVyXAU<_=<#wayL*;_rv zz*cUI3jR@P!l~qMHmY=@+Q?dCxQtP!6ZJ+m%ozA>GPY)8wD335{gH9P9<`rHL`3bz z=HYMlccIJMF1=l+x9jzGgWhh`+fDr6ZZ<}nF}%j`8Kct}T^LgiV>peGVvJN{q!}X} zMr+#hektgwYaE7R7-;C$z z`8+0`9&KdKkh4f-nb8Iig1(Ozh-|53FtqCRI%&(@4jbCj_WNK{9R=%zU%+L};} zFg|AN*wDgfPVF;CMUx?#2+<^H*V`TZX)iLd?|i#^Ds?IuOVr_1riqe_O5OZT7#JoL zgbBWT#vYAHw>b~SjC(ZjZWCs=i5f=q=ozKhnsG$g?x=qgVJ?s!;|U|WX~OO?+RO;! z!Bf2>gS+-}kqLIUHjx&NI12cAPLH_30 zpz(|GKLWpI%lI&UgV?5Q+`AarQDZ94h@7Y~W6w2q)9i?d8u^;C^Cx%=uQ7a4gL6#T zKQc_XcFur-Es?+RnK+e9>LT!oG2>+7X11ap%^``#mc;+-;EP^YCSzpwJItBPO<)Y;(+I*q%5&N{&FsKKx;YGE z#!lSRXVO01#3|jxDc!^=-2{vAKf)Yo;>6u+3<6A#hG0VHKvI$;1DXnIDx|3aO%3X# zO(&CdGFc~`I+>!AsXCddlUX{Mt&@2=S*nvRoovv_CY^Naq*o`~b+S_@yLB?ClOdfP z&?%cv*>x&WrySz#$h+}W3Mb_Ccq&b&GIT0ayd}B9PZj7)L(!RrFrs)yl+1|A8R41{ zbu+?^x7HbJ`;6$srAKF)TxXgHqp5C2v`0mAhR!r+>P$=VnHF}ol+1|A8PPf;x@Lqw zD%xCU+VI&{7Zu)Ao$(TemoR*VI@6vGY`3%B$-ZtIq=2!rubX|n$X+Ay(_apiPzBXG z6UaK_@6wrI>zQELjBx2pDC0C7(t-YKL-N)2#a3dgmPzHRtzBVF6@--iD-&e;}%h^D>h?C z(^CwD)kC`W5LOR5_u{vYIQJ1>KmPsr_v6>kF@E&%v(L|Y{KU)8x%}j#pY!?2M?dHE zlaGGR>n9)m zwczguKLj8M=!>Es`PK}`qHKY;!L^beqa0R02#A3*;A`UlWIfc^pW51@Yl{R8M9 zK>q;x2hcx&{sHt4pnm}U1Lz+>{{Z?2&_96w0rU@`e*pah=pR7;0Qv{eKY;!L^beqa z0R02#A3*;A`UlWIfc^pW51@Yl{R8M9K>q;x2hcx&{sHt4pnm}U1Lz+>{{Z?2(4VDA zrvvC8K>q;x2hg8o^ri#oA4LBk`UlZJi2gzJ52Ak%{e$QqME@ZA2hl%>{z3E)qJI$m zgXkYb{~-DY(Lad(LG%xze-QnH=pRIX9%|D;^bDeB5IuwF8AQ(@dIr%mh@L_845DWc zJ%i{OM9(042GKK!o2U=^%Or(KCpiLG%owXAnJu=ov)MAbJMTGl-r+^bDeB z2t7mS8A8txdWO(5gq|Vv454QTJwxajLeCI-hR`#Fo+0!Mp=Ss^L+BYo&k%Zs&@+Ud zA@mHPX9zt*=ouoPL*#Rad=8P%A@VsyK8Mghg#ID)521eu{X^&h0!aFUSaeKqgNQc!sr!7 zuP}Or(JO59B45MgYnXfuldoa&HB7!n&^Lm<5%i6qZv=fK=o>-L2zo})GlHHG^o*cq z1pOlD7eT)W`bE$$f_@S5F+x5@&@qC35p;`?ZxQs1pkD<2BIp-EzXTybAnpUieSo+R5cdJ{ zYJj{NK!*Wz7(j;sbQnN~0dyE3uLj7g0d)C)sk#sBHnJ^EqnKk)C`n2*0KNC#ThXTz zIf0r)r)ckg2GJW*zN)UZ-d#zNU?6tT0TJiJdt!NFd185Dd185Dd186;HGATD@-=&6 zdh#`U;(GEu`Na0b_QdwY_QdwY_QdwY_QdwY_QdwY_QdwY_QdwY_QdwY_T+2##P{TD z_r&5M?d!9BeYT&^_Vd|(KHJY{`}u4?pY7+feSEe*&-Ulp{yf{CXZ!PPf1d5j^WU9+ zew()DtaiVsb=mo6 zWWt)XraJ$O4q9KV5o^pEw|w5H&l}yh4y`l)`}aTp>#bk^xcq0-Mo0Vq`T5q^KI{zHn&f#$9pRFSI ztgh9w#;pl!(wefStr=_9n(wdw^W(RL{(lX5{O7Gd$NzonwYTw~w=Vzl(YLX=f8Y1E zu=Jm|*3bWa%f`#!YA19(61f#kBc8e{hRv8d4K(UNB?fUWtaNYi~jmCbnxFd^pAhvcVYK` zZm)ko{dfNu+WC3Eqrb4f-esTvy}wI0yZm)emp}Uc{{4E(e*E_9A6MSt_VyRfVd>wI zf7=kvgn#r8*Zy`Zfxc4)eWwgQ{yY-@e(8Sqj{bf5)*1Y(zkVLK{+IICTch6U zEBoI|Pu6pXwZHzem;EpM(4aMBeX+h;!`4WD{k+HiZvQJB@}(ad`+0QxgrV+_p-KPg zS>u)?^z+}7KmI-BLsQ-}_482nw>{loKS92}Xa3=5_}4e^|LpF+-_ZS=``H?r>#tuo zp7-Yb58vj#6Z`wq-++Jjzh1xIa+HVsoc{a~_|NxQe~0}@?ESpgK4jtNNBTEi?5`j8 z)FB-_`n8oM@AG3kwA|nB;PcPV>TiA7hkqR(KZZjqKaXDjSo(qN|ByMn{?q@K>hH6+ ze0h9X`fvGW@I@M0>#v{RCH?>Sy7#yG>#hF2toPT?`{?iEMt}W0YQ9^3zT@YYwQunw zF|^tLP~Yv_^VdIqZf)~N|ETwmxVL-@U;TQ^uK1SW>)+2-fBpL0f8XJ|>F4|UKZn1* zHvK*G4L9VMjGw3e59ZJR{P*|3|Nk>K$S3&-=dfUq3@b{jW&>>*L2}=r5~(FMd2`9o<>)pY^^uhicw; z)0*EJYd*f|2W8vVcZhfVdvC-d-(R+QGIcK;>Cl3m(ionOLx2T zGCpA4bzUY`IxkZTmj6!oIxjP4otJrQVZ++7yuGw!+1~Pf=VkS&^RhP5c{%hsM~9u4 zlS}KN^KyFAc{#uByj)FoUT*vA!uq}Qa_{~&Wq=XFPW zv(BqOps)T+zWNz>^^^JPTkZABspU3K@4SBfYE4-4*1mOMxs6jeuTJ3@`?M~s$Ik1p z+Yj6Bu-kjQ_By;^Em}*K`xy55!*1^s&8t%}ufslPWXu}3)~!uz+w!qd+Zr9RMl7E{ zx?#D$QTsdkSLbzX*m6H(H#pZ*xrnf z%{b zZD-flcGrFFEn7Z!&wcM*TDGz0_Itio`|f|=eeXLi`}TR?=lhlLx^JKMeQ)g_yS2;IB6hg%i-PcMkQvZQi@Td*pi`zyG`Q`hfiKXXo|Nem?G4w(;omAMMYh&wun~ ze6*i_E4=y*@apW-tFue5f84%}+LRMauTCbtI+6711k$S$IIm95ygDiK>SWBTlPIrF zqP#j$^6Es%tCJ$HPG-D1Ve$Hx%{n3R>cqpVlMAm-8oYW!{_2VNt0&p7o@BpzGX3g_ z^y_!q^BdyTlipWPW?wytef4DY)f3QHPb6QTF+E9qeRg=C|LDAV67lxidgpE6zVkLX zV9j^lJo(aWjd~B>|%~>0rx9);<-Ffp<{??nayuW8(dVgC_owsrO zH2%HwHeownkLGQ{{ZIVWd7B)v?9b#$=WWV3p^?)H90-_{+Wbw_=j zZo`q?u-OfJw&93wxSLITu<3SNx0a*7?d|O?>$JaK|DJmJd(||3=WPdj*OznGC+ymg zALzIJ`Oe$HVCU`7t{?e?V{ae3gHsu#QmS(w+mm+i$}|0 zxO9h?zKoZ4;?hoB+A)`L-mWGrpL1n9e(K)7dHdU4=gr^#x9g+M+YRW=bLZ{W0lIbD zTZiw~eLL6wcIR{MAn)A%&i>!q_j_M9FYotu?|$zcEa%?e9^H@2KyQ!U|7c&FAAfT` z`^|aXH|J#E{&b-IEq!xt^3D0kH|HMToL78vj_%Dlxi>#LZ_c;9InVUwe9GJNnf26p z|833MwEpP44|J_F>znmxeeb;c!}{*Y*t^G7??c{pedXOB)c3Dr)@A2?c)j!P1jhTw zbm!f-`nzxS_t9hP)UvJ7d+Wja%X;p-I}`Cf=5}MYHRiU?MkpdW@6I;7`%!pzcH!Mw zh4-F+`w@5_w;g|&-Y5R-yib0yeEy{Sp6po%)=lSq%KrP~^FHnUzV+W{+-Amp`SyRG zx$eBr+TU3pn_cd_&oR#V*xX^~ectEHd*A$)wQt$)dG>kRoWHdkgL(Ts?>H^kUoTt# zzTo4F?t5{z^X|vteVJu>!?KN)LCbxv@UA#^tB%hq^P2ngM@jQx9a-M*2jP8vz_RcD zK)rA9Zn&L4Qtz8?v*q?%w(lGMeaG?LaV&Sot(ngIp4;ttd(Xb@pIVNM3!nY(|95|B z-Vaz0?AO7fW#10m$ANu6urG)1^Kinl&BH0nz8wB;`P?Jh^;hZr$bB6f9jZ?+0Qedd**BB=jQ#=ZLZw@$}zoi|KDuSU!3=A`+sdeZyc{1@4NNi zTl;nESln{@Ie7Qj^Zm}}-r46n<~v{id&lzLF}QcT2itk@xsSfCkM_g0r+1G{-(9bH z_gM4Y^_O?&f8U+seRr<*-TBk^@Ak`i(|6}b-<=PAcb@azpM>`(Ul-@~KAdy<`0cAT zZp~V^oez(}Kb*b!7`U>&Sx?q;=VNfp>RA`oP3Ob)%a1QUHnME(S;y9?<>Mo^GxB%m zW7PJY%=>T>?_<>M{9JsDZgoDKb^CDE?Zbn<4`TZCKZqZFIkPK6(e% zVdrDq`^VkS_(SJo!oE(-S^hog{`8<9Q})jTv=0x^KBm^KBg=hwVD>TX<1;=!v()*R zbKi5`H)s2E_Hll(^Rcks`B<8;91{;z?SG<2^+Oo|R+w{Htv2xb=@XP&U z)!S>kosacNi+{uWH&`}pW7D>_b~+#1j)7m&A0A+R?AW&*$9Bhl?+sd0mVMgWwA^;j zvDou@`|j_+=N;Iu1IO6e!jD78=g__$*|#H~e{6fFwsq>5pW08~iyvn`cIGx`_WjJZ z&V25@x8GRH%vDiw?6OIKDs3T;gbBv zt>b;?zkXML+}p2v-y426e>^yj500bX%^#1B_oMf_7WLub%!lheAFgS9xQ_ARn#6}| z5g#6|d^oTE;e7gsbKxJZ(SJDq{rTIT^?T>j@83^n3qJ>LtiL;-gHzUwHQ)Jkg5}d$ zfKShBKZm||K3xm{{Ib^h^n3WzlbcW1p+7xA`E-r=b7a^Ww|xGH+j-dWIcoc3?rY3; zdrQ`G=X3l^=X1h+PW{pOoL=aBE z*0$yT{qFnRdhUGsKKk7LYWeuK{oZ!l?Y}yo9y+dT96u8V)3A6T|=Za+{Im1v+VrPusGlC*e!7PF`KN8UR{8m-eRS>c({;g5*9AZQ zP5yMv?ej06@1gCd>ujH{qkZ}t{psQD=im0#^|4RiE1#}SeYzI)>AKUWYdD{-;e5It z^XWRwr|U4ylb*IZ)@kcUowjk+X`7RscK+CD7qix(<^A7&vj(g|YsmUyeYJ+I5o^@4 zjc;A6XN_AEmTh?))4okx?&q6rew(xAtp#h*TC$d{6>HU6v(~K*Yt!1Y?B}-~YuDPd z_N@cU{(bB3-?4RKomywsxpiS(T1?-*S=ZK$b!**O_tt~O>X}XZ_J{SS^_TUx_1${1 zo;%IgtX=b8^I!8{^I!A(`ES?!eh0K`{%ihg{%ihgevjMQHUBmLHUBmLHUBlg-x=+i z-}geh=D+5@=D+5@=D+5@=D+6m_pe>^d)C#i`91S$o_V!vevb>=HUBmLHUBmLHUBlg zXJ+l1|C--3w06z!?`XT`zvjQ@zvlP5t6lS7^ZOgy{NDk#YyNBgYyKPl8~z*q8~z*q z8~z*q8-9;V+YSE>{|)~Qzh{H(hX02DhX01&-|lw9f5U&nf5U&nf5U&nf5U&nf5Y#0 zX}jUS;lJVcd$ry0-|*k?-|*k?-|*k?`<>fv`2F5(H~csJH~csJH~csJH~csJH~csJ zH~csJH~csJH~gN9v>W~#evdPn$J@=b+U9q8^Siv=^562`^563N-QN6eZ@2tLu9}gn z=J9#+_`EqM)^7Q4`EU6>W@-Kgv|IjLeve<;E&nb5E&nb5E&nb5E&nb5E&nb5E&naQ z$2;wo-|vrh%kQyI^Vp}|^562`^56117uRn2Z~1TeZ~1TeZ~1TeZ~1TeZ}~lLYPbBi z{I~qK{C+pJTmD;qkEz-nzqy`v$A8Cv$A8Cv$M0`iyW_v(zvI8-zvI8-zvK6Ktljb7 z@!#>^@!#=#-qOqnwL5-i4cZ;Qzm3hzP%|^sJfCTI{CE6!{2tS_JN`R<^F-~A|BnBT z|BnBT|BnBT|BnBT-($db$A8Cv$L}0!^Bk$&@!#>^@!#>^@!#>^@!#>^@p~TC?)dNd zJ)deGOEzaV+CBe0|2_Xb|2_Xbzfrqp)UFw|Yxn&3{P+CMgtU8pFGtz#`S1Df`JE?i z_x$(#_x$(#9;>!{{(Js={(Js=e%CkKJ-^4X&0~jl&wtPF??>~0XV5&?Yo6;hXI$Dn z|2@AmFYTV+89!e=3IXBe5pB~-^}VYvwF>}UNfuL%<47IqndO4%{l&NR}UNfuL%<46>dhK_9vwF>}UNfuL%<47Q1e$XZ&8%Mgo!_ip^LVKJ&Tm$) znbm7%^_p3|W>&A6)oZRHG_!im8N+5)ubI_rX7!p`z2>??GppB}+h}I>npwT(Izuz7 z*UaiQvwF=nhvvCobKRl2?$FHYHM4rntX^{lvYFLuu0u4ldd>60W>&A6)oW(;n&*nm ztX?y#*UaiQvwF>}UNfuL%<8p2_|57yvwF>}UNfuL%<46>dd;j}GppCk>NT@^&8%K? zouir6Yi9MDS-s|(M>DI}%<46>dd;j}GppCk>NT@^&8%KCtJloxHM4r{4}PL8S-oafuelb}T#IRD_1d5OX7!p`y=GRgxjxg(>NT@^&8%KCtJlox zHM4rntX^~NrkT}iX7!rKxy`IzGppDBF4J8 zT64~(|WsHM4%rtY0(h*Ub7gvwqF2Uo-31%=$I6e$Ba_X4bEHUfs<4HM4%rbEan2 zubK60&I2{Ge$A|3GwavP`Zdp|npwYQ)~|i%|IYuN-^^e0Jgb@cYi9nMnZNd(|2w}+ z4bAza<~dk%$)TD3Yi9r2cYd>f?K{8uzxJKq3}Ex}-pw3f`_69`u(^2IT&8HA_cybE z%`9Lu3)sv8HnV`uEMRl_qL~G3E@3pYfXysmbB?xs=Qj)3zVkbu)m+|aW&+!Hesh8C zJHOe$<~&z(o~wQ5HzU~mf1|eV{NMRqIBn(yn|Z-zUa*-LY|e)@7gL*g!De2tnHOx% zk2RN0noB3mbIsAn!T0_#H?vjaZ}BnANp<*0?ocO)v{>97ziGb= z@SFI{0Kcif4Dg%$%K*RWzYOr30L%ctDZmWyJC~dRe$#*%;2+>O6_^2jlYtrFcfKzJ z{I0uXfPa90fZsXjI0qe5gK<7ECI@43FfLcd^k7_{$pF6z!We+a0KapFam^;qTgQ3p z4D!3QnL++R{z3jh{z3jh{y~1{wqw>X&TnUse~^EW-?_&O@;m>SLHq^W{`i7e~^EWe~{k@V$2$5kbjVWklzqt3=zh_Uxfqy>nYfsViwUckw~8^i7=w$Ex0uO_k+&Fm%Miaow+!(Q@f&oD zLAMO?yVM)kv|_X^X4+zyEkpc6{LZyzh~MB^%&Nt7S4^a3h~K1HjF4rB-=*TXCKf{{ zG3gcKTrtEI=V@c)D#oeeQgTd7#avHJOT}bVOh(0!O3XvW+*8a$#gJ1BImK{ShWJf1 z#T-+d?~OU8m}ANi{}BHWzj>yZXNq~Im}iQ4rkH1nbHg#ylp%i2RWw)8T*WnuXs+^w zUvm}BRb1kZ<|<$KHCNGGMROI+RWw)8T*Z0jXs+^wUw0MVRle|l;s3(_ga(-O9%vXNRS6teU^XK`> z|CQfm{y3K&m-_RS|11Aj{;&LB`3(@|EB{x1tyr{T(Te3O|5yI6{9pOM@_*$wE)We_ zG-T0`#qd!KA4NkJ14uDS5CceY?mmW)q9Kbxq`0mc4Oui~(U3($77bZ6WYLi2E5Bi- zeC7YjZyX_C`M>gi<^Rh6mESN^hWQONWtd-k7VTMv`G@&6Xc^`o<{#!C<~R5hqYWA6 zH{K9~>oEuwgX__$#XwXHM8!Z%Gb-Y%~HMRS)C{tj7)UgDF$k6s{t%HG2>%HG2*2x&8Q~w{AK^EG64xPPmM{j< z;(BC8`A7K;rNwwkga#SqALVzgGNb&X{Gwt{=zs zK{G5#_BF@7@@QE;To-{n_wM9GmZf0tj;5krUR^1B9|E`OK5%irbi^1Cjb zE`OK5%Wp;_UH&eAm%q#3GF5^yZl}LF2AA1bosmdUH&eAm%q#3HSZ)8p^)_xKevQOra!6U9t={5}32e~-V%-{bG`8~#j>zsKL> zHwYSopy~0Omr0M`*jRe}3>n7N3Vix`}Z5xh+BPw-Fh zo5_x0+Dz~psLcfb1pfs81pfs81iyKWOz=K^#Px2e)jd8)4pUNcvB)>VT zO!Awh$|S#es!Z}v@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@=x+l@*7Xg zB>yD;B>yD8nXOFnPx4Rl8(+*M|0Mq;znPXy@=x+l@=x-c@yaCsB>yD8!Q@QxPx4Rl zPx4RlPx70G$t3?I{}lfe{}jJL<`|QVG09BvPw^XUj@g-*oyiox`I$`dPw`LjPw`Lj zPw`LjPw|_-&lLX@{}jJj{7msr@teKL6u)`27?jQw{}lfezoF?E&y2z8O!1r1$rS$- z{}jJr>P+!Z@lWxa-^moe8J8?eq4zt?`x6#o?e6u;SxOz}_gPw`LjPw`LjPxDXn zPxDXnPxDXno0ZEnzhUi|naec4o*>iwhPN}#Kg~bQKg~bQKg~bQZ%!oB{L}o?{L}o? z{L}nKPh;pi)BMx?)BMx?X7V!4Kg~bQKg~bQKg~bQKg~bQKg~bQKg~bQKg~bQKg~bQ zKg~bQKg~bQZ@4^$%QMYC%|Fd=-YL`k)BH31GyF6BGyF6BGyF6BW&|_CZ#F73{4@MB z{4@MB{4@MB{4@MUX*0t=!#~46!#~46!#~46!#~4s<}fq-GyF6BGyF6BGyF6BGyF6B zGyF6BhT&sIF*E#Hj?D1S@Xzqi@Xzp@U(5{u4F3%O4F3$jvEIz^&+yOi&+yOi&+r@f z%?$r6|1AG3|1AG3zgfu4^3U?m@|%myEdMOO0p85=&+^am&+^am&+^am&+^am&+^am zo2SeyztQ5%^3U?m@*D8aEdMP3EWcs@%<|9j&+^am&+^am&+^am&+^am&+^am&+^am z&+^am&+^am&+^am&+^am>vl5BuieQkzo;Oy{ImRX{B!(s{B!(9pEJil$3MqE$3Mq! zCNy*WbNqAsbNqAsbNqAsbNqAsbNqAsbNqAsbNqAsbNqAs=1MciKgU1EKgU1EKgU1E zKgU1EZ}v2E{B!(s{B!(s{B!(s{O0sB$3Mq!UN3X}X7)12KgTbc$Q=J1{~Z4u{~W)W zzRdB@@z3$k^Uw1ua5K+8&p*#^?ltrL^ZfJt^ZfJt^ZfJt^ZfJt^ZfJt^ZfJt^ZeSV zm>bMIzuDQ$^Uw3o^Uw3o^Uw2}sm(n9JpVlZJii&+%=6Fl&-0tN%{>1+zm6&M{PX2h33;YZG z3;epQEbuSzFYuco&I11e{{sI4{{sI4zcKtQ@GtN$@GtN$@GtN$@GtN$@GtP2)yo3^ z0{;U40{;U40{;U40>4?lEb=e%FY=3OvdF*4Z`41F{EPgH{AT{L$iK+H$iK+H$iK*M z);Wv(i~NiHi~KTyEb>bQvdF*4zsN7v$s+$E|04e)|04e)zoa0G{EPgH{EPgH{EPgH z{EPhN4ztL=$S*(0BL5=4_$Q0}i~J&>m`%(gzdRv}{EPgH{EPgH{EPfc{7d{|p)Bz) z@k<%9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9#J|M9 z#J|M9#J|LEPCT-SEb%Y#FYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz&FYzz& zFYzz&FYzz&FYzz&FYzz&FY_<+FY_<+FY_<+oAb>w|1$qF|1$qF|1$qF|1$qFzuERI z^P35d&LhkG%lymy%lymy%lymy%lu|VV-`Nk{N~}K3&}G7GXFCFGXFCFGXFCFGXFCF zGXFCFGXFBadDATOFY_<+FY_<+FZ1gfv&_HDzs$ePzs$ePzs$ePzszrbIV=1t{O0nr z!oR}5!oR}5!oR{VSjh^%`PQuPn{mww{|dj^=a{q13jYfK3jYfK3jYfK3jYfK3jYef zIq9tMukf$%ukf$%ukf$%ukf$%ukf3vj%Y1rs9L%D>9L%D>9L%D>9L$}jfGD*r0~D*r0~D*r0~D*r0~D*r0~ zD*r0KIr6OXukx?*uk!0AvdX{8zskSLzsfHp$|}Ek&dl@vrf(@vrf(@r#kN#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD#=pkD z#=pkD#=pkD#=pkD#=pkD#%~rnW}&mjzsA4DuLFtM=&bRt@vrf(@vrggL$by%n93Ue z8vh!YldSWv^RM%-^RM%-^9!&dz{)!R zI{!NVI=`9ztn;t)uk)|-uk)|-uk#BlBihP3|2qFVzkn<2{OkOhmS|eC&M)f9I=|WP ztn;t)uk)K*j&309{OkPd{OkPd{OkPd{G!dQ^P3sZI{!NVI{!NVI{!NVI{!NVI{!NV zI{!NVI{ya$2LA^C2LA^C2LA^C2ERGZ%#cM{DQP>@Ne*& zTh9jn2LA^C2LA^C2LA@Xnf7e(Z}4yMn{Uqs{|5gC{|5gC{|3Lg_iXU%C$ho6!N0+8 zraK$_8~hvm8~hvm8~hvm8~hvmnu~1ki{$)&!C3qo{2Tn6{QC85@^A8Q@(byr(a0wM zCjTb?Cco||oBW&nX34Y3zsbMJzsbMJzsbMJzsWDg%O?LO|0e$?|0e$?|0e$?|0e$? z|0e$?|0e$?|0ci2DVzM8{G0rn{G0rn{G0rn{G0rn{G0rn{G0rn{G0rn{G0rn{CbmY z@^A8Q@^A8Q@^A5P@o({O@o({O@o({O@o({O@o({K53N@bB>N@bB>N@bB>N@bBY8@9^*N>r%4AuXoB0{|>+IDLec-{5$+R{5$+R{5$+R{5$+R z{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{5$+R{JZ>G*zEG}^6&ENVzbM? z%fHLN%fHLN%daoYF28OzyZpQSyZpQSyZpQSyZpQSyZpQSyZpQSyZpQSyZpQSyZpQS zyZpQSyZpQSyZpQSyZqXn?DFsO@AB{R@AB{R@AB{R@AB{R3-YtezstYNzstYNzstYN zzstYNzstYNzstYJzsIjP%O3w8{~rGy{~rGy{~o`dGJE`c{CoU+{CoU+{CoU+{CoU+ z{CoU+{Mxnb@$d2P@oSK>$G^wF$FE1q9>2CNx}@y!@A2>P@A2>P@A2>P@A2>P@A2>P z@A2!H;spn?$L~c4vd6#2zsJAF??nirnadu(ZZ2BT=$^92zsJAFzt6wV@5KtT&%e*V z&+ml`vd_QIzt6wVzt6wVzt6wVzt6A5%Rc`;|31HtI$rD``~3U-`~3U-Ui={Y{9XVd z`~3U-`~3U-`}|%MA^ZIM{QLa-{9Ys>8m(xwqG8QG|33dd|33dd|33dd|33dd|33dd z|33dd|33ddzy3A*{2JKo^Y8QT^B?eQ38Vea0sjI20sjI20sjI20sjI20sjI20sjI2 z0l$VeUbP_y{0IDAy&(tuy4oD@AMhXWAMhXWAMopKbHIPVf53mh?-d_%z^`eH4mSt< z2mA;82mA;82mA;82mA;82mA;82mA;8UNIsE{0IC8{0IC8{0IC8{0IC8{0IEH$!L9Z z$glU!A^#!&A-`8(i>5M%{D=I9{D=G+%N+6_@@s>m4bCC|A^#!&A^#!&A-|?Bhx~{9 z+PWO_AMzjaAMzjaAMzjaAMzjaYwvT&f5?BxugA+F{~`Y&{~^CVFNge^6p5&seY5&seY5&seY5&seY5&seY5x=%E zNBlb39P@j!Necf6jl- zf6jl-f6njq&T`Iw&VSB-&VSB-&hK?0;`LN=&VSB-&abb|IsZAoS2)T!|2h9T|2h9T z|2e+~JLml8{95dIy|SG1pYxydYn^k>ukX(}zt=O5UOQgXJYF>^=lmD^7yK9e7yK9e z7yK9e7yK9e7yK9eUY@Oxc>T<~kebHRVX zf5Csjf5Csjf5GpyWOBiO!GFQ8&yGGj7yK9e7yK9e7yK9e7yO#@T<~A;U+`b>U+`b> zU+{ZPnq2UEU7B3*U+`b>U+`b>U+{a)gIw}o@?Y{_@_Y5DT=HM?U-Dn_dtHQF@?Y{_ z@?Y|MJ&#=SU-Dn_U-J9EF32VSCBIjo$|e6L|0Vw=|0TcIy~`!P*UHHy|0Vw=|0Vw= z|0Vw=|0Vw=|0Vw=|0Vw=|0TcI*vTdTCI2P=CBN6)$tC|K|0Vw=|0Vw=zt`f4*W$?~ z|0Vw=|0TcIBgqy26~9-@$`$_=ztc^p75^3g75^3g75^3g75^3g75^3g75^3g75^3g75^3g z6~9;W$`$_={}ulgzgPFl75^3g75^2#SNqBp{}ulg{}ulg{}ulgzy3e^|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|M&kdZ1L;=qyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2%0L4N&z>HkarU;6*j z|JNYD{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S$o`1Sv#|1bT2>HkarUqk%*|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6**@^|^W{9XPof0w_@-{tS}clo>gUH&eAm%q#3gU4H$4 z>Hn+C-{tS}clo>gUH&eAm%q#3Hkar zU;6**@%Q+9{5^jCf9e0L$FKjd9)FL&$KT`c@%Q+9{5}32e~-V%-{bG`_xOAKJ^mhl zkH5#?@A3Eed;C5A9)FL&$KT`c@%Q+9{5}32e~-V%-{bG` z_xOAKJ^pe2asF}sasF}sasF}sasF|B{eS8IOaEWv{Nw!N{Nw!k|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarUlaTj{1f~W{1f~W{1f~W{1f~W{QCdW|JMZn1pfs8 z1i${j^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2%`e^uO1kKBi0pYixu9@X9ow38f@LbsmUYYyq5mlz1p z9tv7wB*wDn@Q;-kC=jVg9Jn(@-6D1a#IYobl(pPVV>bre!q97=-$I3^Me%%#x%A%8 zkW%9u_}iJ~%+7pwm*jG0o=^J!^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|L;TlKeS)}pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>Hm9d|6}|0|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`2b_2Qo;3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K; z2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L& zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}E z*?(pKmHk)tV*tVcbY=gQ{a5y1*?(pKmHk)tU)g_U|CRk$_Fvh5W&f4^SN31oe`Wub z{TP5S0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_#E9 ze`fzP`=8m50SE&S1|SST&+LC@|1a0CZ#jjr}+F-`Ia+|Bd}O_TSilWB-l)H}>Dye`EiR{WtdC*neaHjr}+F-`Ia+ z|Bd}O_TSilWB-l)H}>Dye`EiR{WtdC*pC4S0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaPpDf2*yd0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaN1#ei}eDfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz8K* z`;!I`4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOhm1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1( zqXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}H zXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMK) z_w1(uj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}H{*C=KfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4Pf84p9U}*z-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaM`S_R|1H0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAVUwEt-T(f*_TG=R|nMgtfPU`P9p_8;v(+JChFX#dgvqy0zw zkM_6Fmvj1fN$^Mi5C;LzK zpX@)`f3p8%|H=N7{U`fR_MhxO*?+SCWdF(jll>?APxhbeKiPk>|78Ek{*(PQfYAU( z0~ifpG=R|nMgtfPU^IZy0CuwfWIqjHG=R|nMg!Q%{*(PD`%m_t>_6Fmvj1fN$^Mi5 zC;LzKpX@)`f3p8%KMi0sfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR*nhB}1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0Q-;j(*Q;T7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!Bb3@BL%z@X;@wzvC%Ghd;X5C7BOI^m3a`T2Xjb#MRX%wPWEnIHY)nIAjzNB`ADFCX&d z31==J>HnX(e5616Z(m&G#fdL2^2M3UMZA2+!Drz6UclqvGw^*c-EsKyUpl{k8T!7T zz#ZJu`+lB1zwaN$^ZN{d^ZNqj^ZPQp^ZR;~^9TMXa{jx18#;feQ#e2N3WD=v567M#d)({% z*n>q5?#p8jyEy!bga6o`ANyC}{8$Knek?aTKh`guAOG0b@3EH*pC5az)%md(&zv87 z1uBkBM%OpKl0Pc`6GY)&mU0<&L3$3&L4SY`uQU-F>!F?PdveYe&X3u zhyUW>xk-o1DUb4>pLl5E{KU^5=O_NbIzRC*!TE`D_WVSWc7E~~ZsUnp%%7il+4uR0 z*T|fodSd7N)Uy=lr=Bi2KlNDe`KbpN&rki2@BGw%Y0gjm410cxws3IQo?iQSPcL`v zsTYc$pL$jA`KgyRou7Ig%lTtZB%eR_Xw&&)|NNgnR=uA;)&!qF_DcBk$6jJ~{@ClD z&Oh`6r-M(}5B*hm{-Mm{{6jDJJ%8fa%kw9mS~`E?IhXS%9=JJw;(z1kPy9%H{^Z(s z;gu(^&sU!1Jzsem=X~Yy`}36t@Xl9$8vK?0om=1j(yf2=oA%Kdxq zynXoXd%t)1okw5$`h&y0Z``?mc<8s^y8X@HKfHeLjaOg!7ytdc?|%8s!<~b-zx(KL z_uy(ye&ulgaPOVN+n3Y7eedvlm$!%CIXpUi?eO)(gBR!S9lmkLrToU_BYN}J?ZY<@ zzkfOP`n|&&hgWa?t6Tr@Hx6&P(Zl%%r$0H|e*1MNzV(S;SBKkvF;PCvXhvz^3`Mb}5^89Cq=YRG5)8{{OnNMGR<)2?Z;`1LI zzPQNY`A-h6_2RuQ`P0j9yg2jImv4T2__@pf#D{%({;v+tKRrDE(TnSU>gNCNmu~&b z%UAr-+jrl%fBS2`v~R!h==K|T4qvH`%MV{@Ja+|IY2VzIx~N!{yCgcjd*|`!C)+c;(kz zt{<(=;4|M}9bFaOq?cOTw)@b%lT z-8tO#$$9YF?XO?%_T>w3`|#lFcV4?|>DO-G|EfFi;N{0VeC6Jww_m?~|L&a!hX;@D zf8(w{M8EhM{QT>3_${9@AKXW|eDupba0{0=-gS8Gwm+<&@BiYn{o;?0-}4)X*ZgsR z`8odl#xGa&x5SGNa9MRZ@s-29N0&c{x9=Y=cjo0q{K@&o;c}mx{l&Xw@ZDR#`q$rk z^4EWM{lBjM{Ihre;;+B^t^fJ#{mQ9`?GgH{p{U8{p=6k td*z>8KAy`TT;bpv2Uj_`&cT%qu61xV@B5x#?>YZ%Z$I_xPY +src/test/resources/opennlp/embeddings/tiny-unigram.model From d423eb27b1284de1d0f179889fd11c23e29fe8d8 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 20:45:49 -0400 Subject: [PATCH 44/82] OPENNLP-1877: Document helpers and validate record and parser boundaries Applies the review conventions from the OPENNLP-1869 review to the embeddings module: private constructors and helpers gain javadoc, the TensorInfo record and FlatJsonFields validate their arguments at the boundary, and the hot-path and test-history commentary shrinks to what the code does. --- .../java/opennlp/embeddings/EmbeddingVocabulary.java | 5 ++--- .../main/java/opennlp/embeddings/FlatJsonFields.java | 6 ++++++ .../java/opennlp/embeddings/SafetensorsFile.java | 1 + .../opennlp/embeddings/SafetensorsHeaderParser.java | 1 + .../opennlp/embeddings/StaticEmbeddingModel.java | 8 ++++++++ .../src/main/java/opennlp/embeddings/TensorInfo.java | 12 ++++++++++++ .../opennlp/embeddings/SafetensorsTestFiles.java | 12 +++++++++--- 7 files changed, 39 insertions(+), 6 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java index 703e8c29ee..1b63e57d19 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java @@ -43,6 +43,7 @@ final class EmbeddingVocabulary { private final Map idByToken; private final List tokenById; + /** Holds the parsed piece-to-row and row-to-piece views; built by the {@code from*} factories. */ private EmbeddingVocabulary(Map idByToken, List tokenById) { this.idByToken = idByToken; this.tokenById = tokenById; @@ -119,9 +120,7 @@ List orderedTokens() { } /** - * Looks up a token's row id. Returns a primitive with a {@code -1} sentinel rather than an - * {@code OptionalInt} because this sits on the per-token hot path of - * {@link StaticEmbeddingModel#embed(String)}. + * Looks up a token's row id. * * @param token The token to look up. Must not be {@code null}. * @return The token's id, or {@code -1} when the token is not in this vocabulary. diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java index 489ebe6afa..5a6ec7a248 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java @@ -44,6 +44,12 @@ private FlatJsonFields() { * @throws IOException Thrown if reading the file fails. */ static Boolean topLevelBoolean(Path file, String field) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (field == null) { + throw new IllegalArgumentException("Field must not be null"); + } final String json = Files.readString(file); final JsonCursor cursor = new JsonCursor(json, file.getFileName().toString()); cursor.skipWhitespace(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 5d8e6d1509..58e7e44b5a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -66,6 +66,7 @@ public final class SafetensorsFile { private final Map tensorsByName; private final Map metadata; + /** Holds the parsed header; built by {@link #read(Path)}. */ private SafetensorsFile(Path file, long dataStart, Map tensorsByName, Map metadata) { this.file = file; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java index a2c1954b66..784a0c754a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java @@ -32,6 +32,7 @@ final class SafetensorsHeaderParser { private final JsonCursor cursor; + /** Wraps the header text in a cursor; driven by {@link #parse(String)}. */ private SafetensorsHeaderParser(String text) { this.cursor = new JsonCursor(text, "safetensors header"); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 37cab554c9..4b45751f7a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -110,6 +110,7 @@ public enum Normalization { private final double[] rowNorms; private final boolean[] specialRows; + /** Holds the loaded, validated state; callers reach this through the {@code load} factories. */ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, EmbeddingVocabulary vocabulary, SubwordTokenizer tokenizer, IntPredicate skipPieceId, boolean normalize, double[] rowNorms, @@ -253,6 +254,13 @@ private static Path firstRegularFile(Path directory, List names) { return null; } + /** + * {@return the named file in the directory, requiring it to exist as a regular file} + * + * @param modelDirectory The model directory. + * @param name The required file name. + * @throws IllegalArgumentException Thrown if the file is absent. + */ private static Path requiredFile(Path modelDirectory, String name) { final Path file = modelDirectory.resolve(name); if (!Files.isRegularFile(file)) { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java index a098d75433..3234a2aa36 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java @@ -38,8 +38,20 @@ public record TensorInfo(String name, String dtype, int[] shape, long dataOffset /** * Creates the metadata, copying {@code shape} so later mutation of the caller's array cannot * corrupt the validated state. + * + * @throws IllegalArgumentException Thrown if {@code name}, {@code dtype}, or {@code shape} is + * {@code null}. */ public TensorInfo { + if (name == null) { + throw new IllegalArgumentException("Name must not be null"); + } + if (dtype == null) { + throw new IllegalArgumentException("Dtype must not be null"); + } + if (shape == null) { + throw new IllegalArgumentException("Shape must not be null"); + } shape = shape.clone(); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java index dd503b9aa4..46ed3d33da 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java @@ -26,12 +26,12 @@ import java.util.StringJoiner; /** - * Writes small well-formed safetensors fixtures for tests and benchmarks, replacing the writer - * that used to be copied into every test class. Negative tests that need deliberately malformed - * bytes still hand-roll them. + * Writes small well-formed safetensors fixtures for tests and benchmarks. Negative tests that + * need deliberately malformed bytes still hand-roll them. */ final class SafetensorsTestFiles { + /** Not instantiable. */ private SafetensorsTestFiles() { } @@ -39,6 +39,12 @@ private SafetensorsTestFiles() { record Tensor(String name, int[] shape, float[] values) { } + /** + * {@return a tensor of the given 2-D matrix, row-major} + * + * @param name The tensor name. + * @param rows The matrix rows, each of the same length. + */ static Tensor matrix(String name, float[][] rows) { final int dimension = rows[0].length; final float[] values = new float[rows.length * dimension]; From 9f7ea98f290222de07945a0c1eb0a88fb2aa781e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 22:29:29 -0400 Subject: [PATCH 45/82] OPENNLP-1877: Let the embeddings JMH benchmark load real model directories The benchmark took only a synthetic fixture. A modelDir parameter now selects the table: the default 'synthetic' keeps the offline fixture, and passing model directories (-p modelDir=dirA,dirB) benchmarks real tables and reports one row each, which is how the potion-vs-bge-m3 comparison is produced. --- .../StaticEmbeddingModelBenchmark.java | 44 ++++++++++++++----- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java index 1ede49337b..b70234cee8 100644 --- a/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java +++ b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java @@ -35,6 +35,7 @@ import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -52,10 +53,15 @@ * JMH benchmark for {@link StaticEmbeddingModel}, the raw-lookup-throughput number the module's * design doc calls for before any "faster than Python" claim is made (a concurrent gRPC-traffic * comparison against a Python baseline is a separate, later benchmark; this one is the JVM-only - * baseline). The fixture is sized to match {@code minishlab/potion-base-8M} (29,528 vocabulary - * rows, 256 dimensions), synthesized rather than downloaded so the benchmark has no network - * dependency, but seeded with real English words so the benchmark sentences tokenize into actual - * vocabulary hits rather than degenerating into all-unknown-token lookups. + * baseline). + * + *

The {@code modelDir} parameter selects the table to benchmark. Its default, + * {@code "synthetic"}, builds a fixture sized to {@code minishlab/potion-base-8M} (29,528 rows, + * 256 dimensions) in a temp directory, so the benchmark runs with no model download. Passing one + * or more real model directories instead, for example + * {@code -p modelDir=/models/potion-base-8M,/models/bge-large-en-v1.5-static}, benchmarks those + * tables directly and reports one row per directory, which is how the two-model comparison in the + * README is produced.

*/ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(TimeUnit.SECONDS) @@ -64,6 +70,9 @@ @Fork(2) public class StaticEmbeddingModelBenchmark { + /** The synthetic-fixture selector; any other value is treated as a model directory path. */ + private static final String SYNTHETIC = "synthetic"; + // Matches minishlab/potion-base-8M's config.json (hidden_dim) and its reported total // parameter count (7,559,168 / 256), verified against the real model repo, not guessed. private static final int VOCAB_SIZE = 29_528; @@ -86,23 +95,36 @@ public class StaticEmbeddingModelBenchmark { @State(Scope.Benchmark) public static class ModelState { + /** + * The model to benchmark: {@code "synthetic"} for the built-in fixture, or a model directory + * path. Override with {@code -p modelDir=dir1,dir2} to benchmark real tables. + */ + @Param({SYNTHETIC}) + public String modelDir; + StaticEmbeddingModel model; private Path tempDir; @Setup(Level.Trial) public void load() throws IOException { - tempDir = Files.createTempDirectory("opennlp-embeddings-jmh"); - final Path vocabFile = writeVocab(tempDir); - final Path safetensorsFile = writeSafetensors(tempDir); - model = StaticEmbeddingModel.load(vocabFile, safetensorsFile, + if (SYNTHETIC.equals(modelDir)) { + tempDir = Files.createTempDirectory("opennlp-embeddings-jmh"); + final Path vocabFile = writeVocab(tempDir); + final Path safetensorsFile = writeSafetensors(tempDir); + model = StaticEmbeddingModel.load(vocabFile, safetensorsFile, Casing.UNCASED, Normalization.L2); + } else { + model = StaticEmbeddingModel.load(Path.of(modelDir)); + } } @TearDown(Level.Trial) public void cleanup() throws IOException { - Files.deleteIfExists(tempDir.resolve("vocab.txt")); - Files.deleteIfExists(tempDir.resolve("model.safetensors")); - Files.deleteIfExists(tempDir); + if (tempDir != null) { + Files.deleteIfExists(tempDir.resolve("vocab.txt")); + Files.deleteIfExists(tempDir.resolve("model.safetensors")); + Files.deleteIfExists(tempDir); + } } private static Path writeVocab(Path dir) throws IOException { From 48e81e5783a9fab3d559cfb82f34f2c17700a245 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 22:31:54 -0400 Subject: [PATCH 46/82] OPENNLP-1877: Add a distillation tutorial and real two-model benchmark numbers TRAINING.md walks through distilling a table from a sentence-transformer teacher, with the multilingual bge-m3 SentencePiece model as the worked example: dimension guidance, assembling the model directory, loading and verifying parity in the JVM, and the WordPiece variant. The README performance section carries real potion-vs-bge-m3 throughput split into the embed and nearest-neighbor cost drivers. --- .../opennlp-embeddings/README.md | 12 ++- .../opennlp-embeddings/TRAINING.md | 97 +++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/TRAINING.md diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md index 969d2eb066..873d575cc6 100644 --- a/opennlp-extensions/opennlp-embeddings/README.md +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -110,9 +110,16 @@ Two seams keep the module small. `SubwordTokenizer` is the tokenization seam: th ## Performance -A static table wins on speed and footprint because there is no model forward pass: the hot path is a vocabulary lookup, a handful of vector adds, and one normalization. The module ships a JMH benchmark (`StaticEmbeddingModelBenchmark`) that measures `embed()` and `mostSimilar()` throughput, so you can reproduce numbers on your own hardware and model. +A static table wins on speed and footprint because there is no model forward pass: the hot path is a vocabulary lookup, a handful of vector adds, and one normalization. The module ships a JMH benchmark (`StaticEmbeddingModelBenchmark`) that measures `embed()` and `mostSimilar()` throughput on a real model directory (`-p modelDir=/path/to/model`), so you can reproduce numbers on your own hardware and model. -In our measurements on the potion-base-8M distilled table, the JVM path ran roughly an order of magnitude faster single-threaded than the model2vec Python reference on the same table, at around a fifth of the resident memory, with output vectors matching the reference within floating-point tolerance. Parity was established before any of the throughput work, so the speed is not bought with accuracy. Treat these as a starting expectation: results depend on the model, the text length distribution, and the hardware, so run the benchmark on the model you plan to use. +Two things drive the numbers, and the benchmark separates them. `embed()` is tokenize-and-pool, so its cost tracks the text and the tokenizer, not the table size. `mostSimilar()` is a brute-force scan over every row, so its cost tracks the vocabulary size directly. A run comparing a small WordPiece table against the large multilingual SentencePiece table makes the split visible (throughput across all cores, one machine, indicative not publishable): + +| table | tokenizer, rows | `embed()` | `mostSimilar()` | +| --- | --- | --- | --- | +| potion-base-8M | WordPiece, 29.5k | ~295k ops/s | ~9,000 ops/s | +| bge-m3 (distilled) | SentencePiece, 250k | ~1.47M ops/s | ~550 ops/s | + +So a large multilingual vocabulary is free for embedding and expensive for a full nearest-neighbor scan; that scan is where an approximate index earns its place once the table is large. Separately, on the potion-base-8M table the JVM path ran roughly an order of magnitude faster single-threaded than the model2vec Python reference at around a fifth of the resident memory, with output vectors matching the reference within floating-point tolerance, so the speed is not bought with accuracy. Treat all of these as a starting expectation and run the benchmark on the model you plan to use. ## Usage @@ -185,5 +192,6 @@ For a multilingual SentencePiece table (for example one distilled from a bge-m3 ## See also +- [`TRAINING.md`](TRAINING.md) for distilling your own table from a sentence-transformer teacher, including the multilingual SentencePiece worked example. - The Dev Manual chapter (`opennlp-docs/src/docbkx/embeddings.xml`) for the same material in the manual. - `opennlp-dl` for the contextual, ONNX-backed sentence vector path, which shares the `TextEmbedder` interface with this module. diff --git a/opennlp-extensions/opennlp-embeddings/TRAINING.md b/opennlp-extensions/opennlp-embeddings/TRAINING.md new file mode 100644 index 0000000000..34269d5e03 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/TRAINING.md @@ -0,0 +1,97 @@ + + +# Distilling a Model for OpenNLP Static Embeddings + +This module loads static embedding tables; it does not produce them. A table is distilled once from a sentence-transformer teacher, offline, in Python, and then loaded in the JVM as many times as you like. This walks through distilling one and assembling the directory `StaticEmbeddingModel.load` expects, using a multilingual SentencePiece model (bge-m3) as the worked example. + +The distillation tool is [Model2Vec](https://github.com/MinishLab/model2vec). It runs the teacher over its own vocabulary once, applies PCA and a Zipf weighting, and writes a flat per-token matrix. There is no training loop and no labelled data; a distillation is minutes on CPU, not hours on a GPU. + +## 1. Set up the distiller + +```bash +uv venv .venv-distill +uv pip install --python .venv-distill "model2vec[distill]" +``` + +## 2. Distill the teacher + +bge-m3 is an XLM-RoBERTa/SentencePiece model with a 250k multilingual vocabulary, native dimension 1024. + +```python +# distill_bge_m3.py +from model2vec.distill import distill + +static = distill("BAAI/bge-m3", pca_dims=256) +static.save_pretrained("bge-m3-static") +print("dim:", static.dim) +``` + +```bash +.venv-distill/bin/python distill_bge_m3.py +``` + +### On the dimension + +`pca_dims` is the one quality knob worth thinking about, and bigger is not better. Distilling bge-m3 at 256 and at 512 gives the same cross-lingual similarity within noise (English/Chinese paraphrase around 0.69 either way), while 512 doubles the matrix on disk and in memory and cuts embedding throughput. PCA to 256 already captures the useful variance of the teacher; the extra dimensions are mostly noise that dilutes the signal. 256 is a good default, and it is where the reference potion tables sit too. + +## 3. Assemble the model directory + +`save_pretrained` writes `model.safetensors`, `tokenizer.json`, and `config.json`, but not the trained SentencePiece `.model` file. That file is what actually segments text, so copy it from the teacher's own repository (on the Hub it is `sentencepiece.bpe.model`) into the same directory: + +```bash +cp bge-m3-tokenizer/sentencepiece.bpe.model bge-m3-static/ +``` + +A loadable SentencePiece directory then holds: + +``` +bge-m3-static/ + sentencepiece.bpe.model # copied from the teacher; segments the text + tokenizer.json # Unigram vocab; its row order maps to the matrix + model.safetensors # the embedding matrix (F16 here, read natively) + config.json # carries "normalize": true|false +``` + +`load` detects the SentencePiece layout from the `.model` file next to `tokenizer.json`; it does not need `tokenizer_config.json`, because the `.model` carries the model's own text normalizer. If you forget the `.model` file, the loader says so by name. + +## 4. Load and verify in the JVM + +```java +StaticEmbeddingModel model = StaticEmbeddingModel.load(Path.of("bge-m3-static")); + +// Multilingual: the same meaning across languages lands nearby. +double crossLingual = model.similarity( + "The weather is beautiful today", "今天天气很好"); // high +double unrelated = model.similarity( + "The weather is beautiful today", "quarterly earnings missed"); // low + +// Sanity: neighbors of a word are its translations and case variants. +model.mostSimilar("coffee", 5); // ▁coffee, ▁Coffee, ▁koffie, ▁kávé, ▁кофе +``` + +Confirm parity against the Python reference before trusting a fresh distillation: embed the same text on both sides and check the vectors match within floating-point tolerance. They should agree to a few parts in ten thousand, because the JVM path reproduces the reference tokenization and pooling exactly, not approximately. + +## The WordPiece path + +A WordPiece teacher (a BERT-family model such as bge-large-en) distills the same way. Its directory layout is the BERT one instead: `vocab.txt` (one token per line, line number is the row), `model.safetensors`, `config.json`, and `tokenizer_config.json` (whose `do_lower_case` sets the casing). `load` detects WordPiece from the presence of `vocab.txt`. + +`save_pretrained` writes `tokenizer.json` rather than a `vocab.txt` for these, so derive `vocab.txt` from the `tokenizer.json` vocabulary in id order, and take `tokenizer_config.json` from the teacher for `do_lower_case`. + +## Where a table's license comes from + +Distillation carries the teacher's license onto the table. bge-m3 is MIT, so its distillation is freely redistributable; a table distilled from a non-commercial or share-alike teacher inherits those terms. Check the teacher before publishing a table. From 6c2a938a5fbeff51908dffe4363d3cff3263c43d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 22:42:15 -0400 Subject: [PATCH 47/82] OPENNLP-1877: Load WordPiece tables that dropped the frame tokens Model2Vec mean-pools content pieces and never frames, so it removes [CLS]/[SEP] from the distilled vocabulary, keeping only [PAD]/[UNK]. Such tables could not load because the WordPiece encoder requires the frame tokens. The loader now caches the absent frame tokens onto the unknown row: the encoder still frames, and pooling skips the frame by id exactly as before, so which pieces are pooled does not change. [PAD] and [MASK] join the neighbor-exclusion set, since a distilled table keeps rows for them that text never tokenizes to. --- .../embeddings/StaticEmbeddingModel.java | 59 ++++++++++++++++--- .../embeddings/StaticEmbeddingModelTest.java | 50 ++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 4b45751f7a..13962f4538 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -19,7 +19,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -88,10 +90,12 @@ public enum Normalization { private static final List SENTENCEPIECE_MODEL_FILE_NAMES = List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); private static final int[] NO_EXCLUDED_ROWS = new int[0]; - // Never meaningful as a "similar word" result. + // Never meaningful as a "similar word" result. Includes [PAD] and [MASK], which a distilled + // table keeps although text never tokenizes to them, so they would otherwise surface as + // neighbors. private static final Set WORDPIECE_SPECIAL_TOKENS = Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, - WordpieceTokenizer.BERT_UNK_TOKEN); + WordpieceTokenizer.BERT_UNK_TOKEN, "[PAD]", "[MASK]"); private static final Set SENTENCEPIECE_SPECIAL_TOKENS = Set.of("", "", "", "", ""); @@ -277,7 +281,9 @@ private static Path requiredFile(Path modelDirectory, String name) { * * @param vocabularyFile The {@code vocab.txt} file: one token per line, line number is the * token's row id. Must not be {@code null}, must exist, and must - * contain the {@code [CLS]}, {@code [SEP]}, and {@code [UNK]} tokens. + * contain the {@code [UNK]} token. The {@code [CLS]} and {@code [SEP]} + * frame tokens are optional: a distilled table that dropped them (as + * Model2Vec does) still loads, because the frame is never pooled. * @param safetensorsFile The {@code model.safetensors} file. Must not be {@code null} and * must exist, and must contain exactly one 2-D float tensor * (the embedding matrix) whose row count matches the vocabulary size. @@ -310,20 +316,59 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromVocabTxt(vocabularyFile); final Matrix matrix = readMatrix(vocabulary, safetensorsFile, vocabularyFile.toString()); + final int unknownId = vocabulary.id(WordpieceTokenizer.BERT_UNK_TOKEN); + if (unknownId < 0) { + throw new IllegalArgumentException("Vocabulary " + vocabularyFile + " has no " + + WordpieceTokenizer.BERT_UNK_TOKEN + " token; a WordPiece embedding model needs an " + + "unknown token as the fallback for out-of-vocabulary text"); + } final WordpieceEncoder tokenizer = - new WordpieceEncoder(vocabulary.orderedTokens(), casing == Casing.UNCASED); - // The encoder validated the frame tokens' presence, so these rows exist. + wordpieceEncoder(vocabulary, casing == Casing.UNCASED, unknownId); + // The encoder frames every encoding with [CLS] ... [SEP], and pooling skips that frame. When + // the distillation kept the frame rows, skip them by their own ids; when it dropped them, + // wordpieceEncoder framed with the unknown id instead, so skipping the unknown id removes + // them. A negative id is the "absent" sentinel and matches no emitted piece. final int classificationId = vocabulary.id(WordpieceTokenizer.BERT_CLS_TOKEN); final int separatorId = vocabulary.id(WordpieceTokenizer.BERT_SEP_TOKEN); - final int unknownId = vocabulary.id(WordpieceTokenizer.BERT_UNK_TOKEN); final IntPredicate skipPieceId = - id -> id == classificationId || id == separatorId || id == unknownId; + id -> id == unknownId || id == classificationId || id == separatorId; return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size()), specialRows(vocabulary, WORDPIECE_SPECIAL_TOKENS)); } + /** + * Builds the WordPiece encoder, caching {@code [CLS]} and {@code [SEP]} onto the unknown row + * when the distilled vocabulary dropped them. A static embedding table mean-pools its content + * pieces and never frames, so distillers routinely remove {@code [CLS]}/{@code [SEP]} from the + * table; the encoder still frames every encoding and needs an id for the frame, and pooling + * skips the frame regardless of its id, so pointing the absent frame tokens at the unknown row + * makes the model loadable without changing which pieces are pooled. + * + * @param vocabulary The matrix row vocabulary; must contain the unknown token. + * @param lowerCase Whether the tokenizer lower-cases and strips accents. + * @param unknownId The unknown token's row, reused as the frame id when a frame token is + * absent. + * @return The encoder. + */ + private static WordpieceEncoder wordpieceEncoder(EmbeddingVocabulary vocabulary, + boolean lowerCase, int unknownId) { + if (vocabulary.id(WordpieceTokenizer.BERT_CLS_TOKEN) >= 0 + && vocabulary.id(WordpieceTokenizer.BERT_SEP_TOKEN) >= 0) { + return new WordpieceEncoder(vocabulary.orderedTokens(), lowerCase); + } + final List tokens = vocabulary.orderedTokens(); + final Map ids = new HashMap<>(tokens.size() * 2); + for (int id = 0; id < tokens.size(); id++) { + ids.put(tokens.get(id), id); + } + ids.putIfAbsent(WordpieceTokenizer.BERT_CLS_TOKEN, unknownId); + ids.putIfAbsent(WordpieceTokenizer.BERT_SEP_TOKEN, unknownId); + return new WordpieceEncoder(ids, lowerCase, WordpieceTokenizer.BERT_CLS_TOKEN, + WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN); + } + /** * Loads a SentencePiece static embedding model from a trained SentencePiece {@code .model} * file, the Unigram {@code tokenizer.json} naming the matrix rows, and a safetensors weight diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index da17b2c656..fc1f0c0196 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -104,6 +104,56 @@ void testLoadsAnF16EmbeddingMatrix(@TempDir Path dir) throws IOException { assertArrayEquals(new float[] {3.5f, 35f, 350f}, model.embed("hello world"), 1e-2f); } + @Test + void testLoadsAModelWhoseVocabularyDroppedTheFrameTokens(@TempDir Path dir) throws IOException { + // Model2Vec mean-pools content pieces and never frames, so it removes [CLS]/[SEP] from the + // distilled table, keeping only [PAD]/[UNK]. Such a table must still load; the loader caches + // the frame onto the unknown row and pooling skips it. The content rows below carry the same + // values as the framed fixture, so the embedding must match it piece for piece. + final List tokens = List.of("[PAD]", "[UNK]", "hello", "world", "cat"); + final float[][] rows = { + {9f, 9f, 9f}, // [PAD], never pooled + {8f, 8f, 8f}, // [UNK], never pooled + {3f, 30f, 300f}, // hello, same as the framed fixture's row + {4f, 40f, 400f}, // world, same as the framed fixture's row + {5f, 50f, 500f}, // cat, same as the framed fixture's row + }; + final Path vocab = dir.resolve("vocab.txt"); + Files.write(vocab, tokens); + final Path tensors = dir.resolve("model.safetensors"); + SafetensorsTestFiles.write(tensors, SafetensorsTestFiles.matrix("embeddings", rows)); + + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(vocab, tensors, Casing.UNCASED, Normalization.NONE); + + // (hello + world) / 2, identical to testEmbedMeanPoolsWithoutWeights: the cached frame and + // any [UNK] are skipped, so only the two content pieces pool. + assertArrayEquals(new float[] {3.5f, 35f, 350f}, model.embed("hello world"), 1e-5f); + // "xyzzy" folds to [UNK] and is dropped, leaving just "cat". + assertArrayEquals(new float[] {5f, 50f, 500f}, model.embed("cat xyzzy"), 1e-5f); + // Text with no content pieces is a zero vector, not the frame or [UNK] vector. + assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed("xyzzy"), 1e-5f); + // The unknown row must never surface as a neighbor. + for (final Neighbor neighbor : model.mostSimilar("cat", 4)) { + assertTrue(!"[UNK]".equals(neighbor.token()) && !"[PAD]".equals(neighbor.token()), + "a special row leaked into neighbors: " + neighbor.token()); + } + } + + @Test + void testRejectsAWordPieceVocabularyWithoutUnknownToken(@TempDir Path dir) throws IOException { + final List tokens = List.of("[CLS]", "[SEP]", "hello", "world"); + final float[][] rows = {{0f, 0f, 0f}, {1f, 1f, 1f}, {2f, 2f, 2f}, {3f, 3f, 3f}}; + final Path vocab = dir.resolve("vocab.txt"); + Files.write(vocab, tokens); + final Path tensors = dir.resolve("model.safetensors"); + SafetensorsTestFiles.write(tensors, SafetensorsTestFiles.matrix("embeddings", rows)); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> StaticEmbeddingModel.load(vocab, tensors, Casing.UNCASED, Normalization.NONE)); + assertTrue(e.getMessage().contains("[UNK]"), e.getMessage()); + } + @Test void testEmbedAppliesPerTokenWeightsButDividesByTokenCount(@TempDir Path dir) throws IOException { From 2ae49124f5d1062650596552545e06a98a6eb8b9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 23:06:30 -0400 Subject: [PATCH 48/82] OPENNLP-1877: Add the AssembleModel CLI to complete and verify distilled directories A Model2Vec distillation writes model.safetensors, tokenizer.json, and config.json, but not the vocab.txt and tokenizer_config.json a WordPiece model needs, nor the trained SentencePiece .model file. AssembleModel completes a WordPiece directory by deriving the two missing files from tokenizer.json (the vocabulary in id order, the casing from the normalizer's lowercase flag) and reports the SentencePiece .model file it cannot fabricate, then loads the result to verify it. Wired as its own opennlp-embeddings command with a launcher script and distribution entry, mirroring the spellcheck module. --- opennlp-distr/src/main/assembly/bin.xml | 7 + .../opennlp-embeddings/TRAINING.md | 10 + opennlp-extensions/opennlp-embeddings/pom.xml | 5 + .../src/main/bin/embeddings | 56 +++ .../src/main/bin/embeddings.bat | 51 +++ .../opennlp/embeddings/ModelAssembler.java | 418 ++++++++++++++++++ .../cmdline/AssembleModelParams.java | 34 ++ .../embeddings/cmdline/AssembleModelTool.java | 78 ++++ .../java/opennlp/embeddings/cmdline/CLI.java | 131 ++++++ .../embeddings/ModelAssemblerTest.java | 206 +++++++++ 10 files changed, 996 insertions(+) create mode 100755 opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings.bat create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java diff --git a/opennlp-distr/src/main/assembly/bin.xml b/opennlp-distr/src/main/assembly/bin.xml index 0981003467..ad465a18d6 100644 --- a/opennlp-distr/src/main/assembly/bin.xml +++ b/opennlp-distr/src/main/assembly/bin.xml @@ -107,6 +107,13 @@ bin + + ../opennlp-extensions/opennlp-embeddings/src/main/bin + 755 + 755 + bin + + ../opennlp-tools/lang 644 diff --git a/opennlp-extensions/opennlp-embeddings/TRAINING.md b/opennlp-extensions/opennlp-embeddings/TRAINING.md index 34269d5e03..450032f5ed 100644 --- a/opennlp-extensions/opennlp-embeddings/TRAINING.md +++ b/opennlp-extensions/opennlp-embeddings/TRAINING.md @@ -69,6 +69,16 @@ bge-m3-static/ `load` detects the SentencePiece layout from the `.model` file next to `tokenizer.json`; it does not need `tokenizer_config.json`, because the `.model` carries the model's own text normalizer. If you forget the `.model` file, the loader says so by name. +### Let the tool assemble it + +Rather than assemble the directory by hand, run the `AssembleModel` command. It completes the directory in place and verifies it by loading it, so a run that prints a summary is a directory that works: + +``` +opennlp-embeddings AssembleModel -modelDir bge-m3-static +``` + +For a WordPiece distillation it derives the missing `vocab.txt` and `tokenizer_config.json` from `tokenizer.json` (the row order is the vocabulary in id order; the casing is the normalizer's lowercase flag). For a SentencePiece distillation it checks that the trained `.model` file is present and names the fix if it is not. Either way it prints the family, row count, and dimension of the loaded model. + ## 4. Load and verify in the JVM ```java diff --git a/opennlp-extensions/opennlp-embeddings/pom.xml b/opennlp-extensions/opennlp-embeddings/pom.xml index 94d9907352..7e827cf597 100644 --- a/opennlp-extensions/opennlp-embeddings/pom.xml +++ b/opennlp-extensions/opennlp-embeddings/pom.xml @@ -47,6 +47,11 @@ opennlp-subword + + org.apache.opennlp + opennlp-cli + + org.junit.jupiter junit-jupiter-api diff --git a/opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings b/opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings new file mode 100755 index 0000000000..3a105a7ce2 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings @@ -0,0 +1,56 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Note: Do not output anything in this script file, any output +# may be inadvertantly placed in any output files if +# output redirection is used. + +# determine OPENNLP_HOME - $0 may be a symlink to OpenNLP's home +PRG="$0" + +while [ -h "$PRG" ] ; do + ls=$(ls -ld "$PRG") + link=$(expr "$ls" : '.*-> \(.*\)$') + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="$(dirname "$PRG")/$link" + fi +done + +saveddir=$(pwd) + +OPENNLP_HOME=$(dirname "$PRG")/.. + +# make it fully qualified +OPENNLP_HOME=$(cd "$OPENNLP_HOME" && pwd) + +cd "$saveddir" || exit + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + JAVACMD="$JAVA_HOME/bin/java" + else + JAVACMD="$(which java)" + fi +fi + +CLASSPATH=$(echo "$OPENNLP_HOME"/lib/*.jar | tr ' ' ':') + +$JAVACMD -Xmx1024m -Dlog4j.configurationFile="$OPENNLP_HOME/conf/log4j2.xml" -cp "$CLASSPATH" opennlp.embeddings.cmdline.CLI "$@" diff --git a/opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings.bat b/opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings.bat new file mode 100644 index 0000000000..199d5820ff --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/bin/embeddings.bat @@ -0,0 +1,51 @@ +@ECHO off + +REM # Licensed to the Apache Software Foundation (ASF) under one +REM # or more contributor license agreements. See the NOTICE file +REM # distributed with this work for additional information +REM # regarding copyright ownership. The ASF licenses this file +REM # to you under the Apache License, Version 2.0 (the +REM # "License"); you may not use this file except in compliance +REM # with the License. You may obtain a copy of the License at +REM # +REM # http://www.apache.org/licenses/LICENSE-2.0 +REM # +REM # Unless required by applicable law or agreed to in writing, +REM # software distributed under the License is distributed on an +REM # # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +REM # KIND, either express or implied. See the License for the +REM # specific language governing permissions and limitations +REM # under the License. + +REM # Note: Do not output anything in this script file, any output +REM # may be inadvertantly placed in any output files if +REM # output redirection is used. +SETLOCAL + +IF "%JAVA_CMD%" == "" ( + IF "%JAVA_HOME%" == "" ( + SET JAVA_CMD=java + ) ELSE ( + REM # Keep JAVA_HOME to short-name without spaces + FOR %%A IN ("%JAVA_HOME%") DO SET JAVA_CMD=%%~sfA\bin\java + ) +) + +REM # Should work with Windows XP and greater. If not, specify the path to where it is installed. +IF "%OPENNLP_HOME%" == "" ( + SET OPENNLP_HOME=%~sp0.. +) ELSE ( + REM # Keep OPENNLP_HOME to short-name without spaces + FOR %%A IN ("%OPENNLP_HOME%") DO SET OPENNLP_HOME=%%~sfA +) +setLocal EnableDelayedExpansion +set CLASSPATH=" + +FOR %%A IN ("%OPENNLP_HOME%\lib\*.jar") DO ( + set CLASSPATH=!CLASSPATH!;%%A +) +set CLASSPATH=!CLASSPATH!" + +%JAVA_CMD% -Xmx1024m "-Dlog4j.configurationFile=%OPENNLP_HOME%\conf\log4j2.xml" -cp %CLASSPATH% opennlp.embeddings.cmdline.CLI %* + +ENDLOCAL diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java new file mode 100644 index 0000000000..06fc0997c6 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java @@ -0,0 +1,418 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Turns a distilled model directory (the layout the Model2Vec {@code save_pretrained} writes) into + * a directory {@link StaticEmbeddingModel#load(Path)} can open, then verifies it by loading it. + * + *

A distillation ships {@code model.safetensors}, {@code tokenizer.json}, and + * {@code config.json}, but not the two files the loader also needs for a WordPiece model + * ({@code vocab.txt} and {@code tokenizer_config.json}), and not the trained SentencePiece + * {@code .model} file. This class fills the WordPiece gap from {@code tokenizer.json} itself: the + * matrix row order is the {@code model.vocab} dictionary in id order, and the casing is the + * {@code normalizer.lowercase} flag. It cannot fabricate the SentencePiece {@code .model} file, + * which comes from the teacher, so it reports that as an actionable error.

+ * + *

Assembly writes only the missing files and never overwrites an existing one, so a directory + * a caller already completed by hand is left intact.

+ */ +public final class ModelAssembler { + + private static final String SAFETENSORS_FILE_NAME = "model.safetensors"; + private static final String TOKENIZER_JSON_FILE_NAME = "tokenizer.json"; + private static final String CONFIG_FILE_NAME = "config.json"; + private static final String VOCABULARY_FILE_NAME = "vocab.txt"; + private static final String TOKENIZER_CONFIG_FILE_NAME = "tokenizer_config.json"; + private static final List SENTENCEPIECE_MODEL_FILE_NAMES = + List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); + + private ModelAssembler() { + } + + /** + * The outcome of assembling a directory: what family it is, the files that were written, and the + * stats read back from the loaded model. + * + * @param family {@code "WordPiece"} or {@code "SentencePiece"}. + * @param dimension The embedding dimension of the loaded model. + * @param vocabularySize The number of rows in the loaded model's table. + * @param wroteVocabulary Whether a {@code vocab.txt} was written. + * @param wroteTokenizerConfig Whether a {@code tokenizer_config.json} was written. + */ + public record Result(String family, int dimension, int vocabularySize, + boolean wroteVocabulary, boolean wroteTokenizerConfig) { + } + + /** + * Assembles and verifies a model directory in place. + * + * @param modelDirectory The distilled model directory. Must not be {@code null} and must be a + * directory holding at least {@code model.safetensors}, + * {@code tokenizer.json}, and {@code config.json}. + * @return The assembly result. + * @throws IllegalArgumentException Thrown if {@code modelDirectory} is {@code null}, is not a + * directory, is missing a required distillation file, is a SentencePiece model without its + * {@code .model} file, or does not load after assembly. + * @throws IOException Thrown if reading or writing a file fails. + */ + public static Result assemble(Path modelDirectory) throws IOException { + if (modelDirectory == null) { + throw new IllegalArgumentException("ModelDirectory must not be null"); + } + if (!Files.isDirectory(modelDirectory)) { + throw new IllegalArgumentException( + "Model directory does not exist or is not a directory: " + modelDirectory); + } + requireFile(modelDirectory, SAFETENSORS_FILE_NAME); + requireFile(modelDirectory, CONFIG_FILE_NAME); + final Path tokenizerJson = requireFile(modelDirectory, TOKENIZER_JSON_FILE_NAME); + + final TokenizerJson tokenizer = readTokenizerJson(tokenizerJson); + return switch (tokenizer.modelType()) { + case "WordPiece" -> assembleWordpiece(modelDirectory, tokenizer); + case "Unigram" -> assembleSentencePiece(modelDirectory); + default -> throw new IllegalArgumentException(tokenizerJson + " has a '" + + tokenizer.modelType() + "' tokenizer model; only WordPiece and Unigram " + + "(SentencePiece) distillations are supported"); + }; + } + + /** + * Assembles a WordPiece directory, deriving {@code vocab.txt} and {@code tokenizer_config.json} + * from {@code tokenizer.json} when they are absent, then loading to verify. + * + * @param modelDirectory The model directory. + * @param tokenizer The parsed {@code tokenizer.json}. + * @return The assembly result. + * @throws IOException Thrown if reading or writing a file fails. + */ + private static Result assembleWordpiece(Path modelDirectory, TokenizerJson tokenizer) + throws IOException { + final Path vocabularyFile = modelDirectory.resolve(VOCABULARY_FILE_NAME); + boolean wroteVocabulary = false; + if (!Files.exists(vocabularyFile)) { + if (tokenizer.orderedVocabulary() == null) { + throw new IllegalArgumentException("tokenizer.json in " + modelDirectory + + " has no model.vocab dictionary; cannot derive " + VOCABULARY_FILE_NAME); + } + Files.write(vocabularyFile, tokenizer.orderedVocabulary()); + wroteVocabulary = true; + } + final Path tokenizerConfigFile = modelDirectory.resolve(TOKENIZER_CONFIG_FILE_NAME); + boolean wroteTokenizerConfig = false; + if (!Files.exists(tokenizerConfigFile)) { + // The BERT normalizer's lowercase flag is the casing; default to lower-casing (the uncased + // convention) when the tokenizer does not state it, which the load then reads back. + final boolean lowerCase = tokenizer.lowerCase() == null || tokenizer.lowerCase(); + Files.writeString(tokenizerConfigFile, + "{\n \"do_lower_case\": " + lowerCase + "\n}\n", StandardCharsets.UTF_8); + wroteTokenizerConfig = true; + } + final StaticEmbeddingModel model = load(modelDirectory); + return new Result("WordPiece", model.dimension(), model.vocabularySize(), + wroteVocabulary, wroteTokenizerConfig); + } + + /** + * Assembles a SentencePiece directory: it only needs the trained {@code .model} file to be + * present, which the distillation does not ship, so a missing one is an actionable error. + * + * @param modelDirectory The model directory. + * @return The assembly result. + * @throws IOException Thrown if loading fails to read a file. + */ + private static Result assembleSentencePiece(Path modelDirectory) throws IOException { + if (firstExisting(modelDirectory, SENTENCEPIECE_MODEL_FILE_NAMES) == null) { + throw new IllegalArgumentException("Model directory " + modelDirectory + " is a " + + "SentencePiece model but has no trained model file (one of " + + String.join(", ", SENTENCEPIECE_MODEL_FILE_NAMES) + "); copy it from the teacher " + + "model's repository (it is named sentencepiece.bpe.model there) into this directory"); + } + final StaticEmbeddingModel model = load(modelDirectory); + return new Result("SentencePiece", model.dimension(), model.vocabularySize(), false, false); + } + + /** + * Loads the assembled directory to verify it, translating a load failure into an assembly + * failure with the same message. + * + * @param modelDirectory The assembled directory. + * @return The loaded model. + * @throws IOException Thrown if reading a file fails. + */ + private static StaticEmbeddingModel load(Path modelDirectory) throws IOException { + try { + return StaticEmbeddingModel.load(modelDirectory); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Assembled directory " + modelDirectory + + " does not load: " + e.getMessage(), e); + } + } + + /** + * {@return the required file in the directory} + * + * @param directory The model directory. + * @param name The required file name. + * @throws IllegalArgumentException Thrown if the file is absent. + */ + private static Path requireFile(Path directory, String name) { + final Path file = directory.resolve(name); + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("Model directory " + directory + " has no " + name + + "; it does not look like a distilled model directory"); + } + return file; + } + + /** + * {@return the first of the given names that exists in the directory, or {@code null}} + * + * @param directory The directory to look in. + * @param names The names to try, in order. + */ + private static Path firstExisting(Path directory, List names) { + for (final String name : names) { + final Path file = directory.resolve(name); + if (Files.isRegularFile(file)) { + return file; + } + } + return null; + } + + /** + * The fields read out of a {@code tokenizer.json} for assembly. + * + * @param modelType The {@code model.type}, e.g. {@code "WordPiece"} or {@code "Unigram"}. + * @param orderedVocabulary The matrix row order for a WordPiece dictionary vocabulary, or + * {@code null} when the model is not a WordPiece dictionary. + * @param lowerCase The {@code normalizer.lowercase} flag, or {@code null} when absent. + */ + private record TokenizerJson(String modelType, List orderedVocabulary, + Boolean lowerCase) { + } + + /** + * Reads the {@code model.type}, the WordPiece {@code model.vocab} dictionary in id order, and the + * {@code normalizer.lowercase} flag out of a {@code tokenizer.json}. + * + * @param file The {@code tokenizer.json} file. + * @return The parsed fields. + * @throws IllegalArgumentException Thrown if the file is not a well-formed {@code tokenizer.json}. + * @throws IOException Thrown if reading the file fails. + */ + private static TokenizerJson readTokenizerJson(Path file) throws IOException { + final String json = Files.readString(file); + final JsonCursor cursor = new JsonCursor(json, file.getFileName().toString()); + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + String modelType = null; + List orderedVocabulary = null; + Boolean lowerCase = null; + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "model" -> { + final ModelSection model = parseModel(cursor); + modelType = model.type(); + orderedVocabulary = model.orderedVocabulary(); + } + case "normalizer" -> lowerCase = parseNormalizerLowercase(cursor); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a field, got '" + next + "'"); + } + } + cursor.requireEnd("Trailing content after the top-level object"); + if (modelType == null) { + throw new IllegalArgumentException(file + " has no model.type"); + } + return new TokenizerJson(modelType, orderedVocabulary, lowerCase); + } + + /** The {@code model} object's type and, for a WordPiece dictionary, its rows in id order. */ + private record ModelSection(String type, List orderedVocabulary) { + } + + /** + * Parses the {@code model} object for its {@code type} and, when the vocabulary is a WordPiece + * dictionary, its rows in id order. + * + * @param cursor The cursor, positioned at the object's opening brace. + * @return The parsed type and, for a dictionary vocabulary, the ordered rows. + */ + private static ModelSection parseModel(JsonCursor cursor) { + cursor.expect('{'); + cursor.skipWhitespace(); + String type = null; + List orderedVocabulary = null; + if (cursor.peek() == '}') { + cursor.consume(); + return new ModelSection(null, null); + } + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if ("type".equals(key)) { + type = cursor.parseString(); + } else if ("vocab".equals(key) && cursor.peek() == '{') { + orderedVocabulary = parseVocabularyDictionary(cursor); + } else { + cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return new ModelSection(type, orderedVocabulary); + } + throw cursor.malformed("Expected ',' or '}' after a model field, got '" + next + "'"); + } + } + + /** + * Parses a WordPiece {@code vocab} dictionary of {@code "token": id} pairs into the token list in + * id order. + * + * @param cursor The cursor, positioned at the dictionary's opening brace. + * @return The tokens in id order. + * @throws IllegalArgumentException Thrown if an id repeats or the ids are not a gapless range. + */ + private static List parseVocabularyDictionary(JsonCursor cursor) { + cursor.expect('{'); + cursor.skipWhitespace(); + final Map tokenById = new LinkedHashMap<>(); + if (cursor.peek() == '}') { + cursor.consume(); + return List.of(); + } + while (true) { + cursor.skipWhitespace(); + final String token = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + final long id = cursor.parseLong(); + if (tokenById.putIfAbsent(id, token) != null) { + throw cursor.malformed("Vocabulary id " + id + " is assigned more than once"); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a vocab entry, got '" + next + "'"); + } + final List> entries = new ArrayList<>(tokenById.entrySet()); + entries.sort(Comparator.comparingLong(Map.Entry::getKey)); + final List ordered = new ArrayList<>(entries.size()); + for (int row = 0; row < entries.size(); row++) { + final Map.Entry entry = entries.get(row); + if (entry.getKey() != row) { + throw cursor.malformed("Vocabulary ids are not a gapless range: expected id " + row + + " but found " + entry.getKey()); + } + ordered.add(entry.getValue()); + } + return ordered; + } + + /** + * Reads the flat {@code lowercase} boolean of a {@code normalizer} object, for the BERT + * normalizer a WordPiece distillation carries. + * + * @param cursor The cursor, positioned at the normalizer value. + * @return The {@code lowercase} flag, or {@code null} when the value is JSON null or the flag is + * absent (for example a nested normalizer with no flat flag). + */ + private static Boolean parseNormalizerLowercase(JsonCursor cursor) { + if (cursor.peek() != '{') { + cursor.skipValue(); + return null; + } + cursor.expect('{'); + cursor.skipWhitespace(); + Boolean lowerCase = null; + if (cursor.peek() == '}') { + cursor.consume(); + return null; + } + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if ("lowercase".equals(key)) { + if (cursor.consumeLiteral("true")) { + lowerCase = Boolean.TRUE; + } else if (cursor.consumeLiteral("false")) { + lowerCase = Boolean.FALSE; + } else { + cursor.skipValue(); + } + } else { + cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return lowerCase; + } + throw cursor.malformed("Expected ',' or '}' after a normalizer field, got '" + next + "'"); + } + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java new file mode 100644 index 0000000000..5f23d332d1 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings.cmdline; + +import java.io.File; + +import opennlp.tools.cmdline.ArgumentParser.ParameterDescription; + +/** + * The command-line arguments of {@link AssembleModelTool}. + */ +interface AssembleModelParams { + + /** + * {@return the distilled model directory to assemble in place and verify} + */ + @ParameterDescription(valueName = "dir", + description = "the distilled model directory to complete in place and verify") + File getModelDir(); +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java new file mode 100644 index 0000000000..7856064897 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings.cmdline; + +import java.io.File; +import java.io.IOException; + +import opennlp.embeddings.ModelAssembler; +import opennlp.tools.cmdline.BasicCmdLineTool; +import opennlp.tools.cmdline.TerminateToolException; + +/** + * Completes a distilled embedding model directory so {@code StaticEmbeddingModel.load} can open it, + * then verifies it by loading it. + * + *

A Model2Vec distillation writes {@code model.safetensors}, {@code tokenizer.json}, and + * {@code config.json}. For a WordPiece model this tool derives the missing {@code vocab.txt} and + * {@code tokenizer_config.json} from {@code tokenizer.json}. For a SentencePiece model it checks + * that the trained {@code .model} file, which comes from the teacher, is present, and it names the + * fix if it is not. Either way it loads the assembled directory and prints its family, dimension, + * and vocabulary size, so a run that prints a summary is a directory that works.

+ */ +public class AssembleModelTool extends BasicCmdLineTool { + + interface Params extends AssembleModelParams { + } + + @Override + public String getShortDescription() { + return "Completes and verifies a distilled static embedding model directory"; + } + + @Override + public String getHelp() { + return getBasicHelp(Params.class); + } + + @Override + public void run(String[] args) { + final Params params = validateAndParseParams(args, Params.class); + final File modelDir = params.getModelDir(); + if (!modelDir.isDirectory()) { + throw new TerminateToolException(1, + "Model directory does not exist or is not a directory: " + modelDir); + } + final ModelAssembler.Result result; + try { + result = ModelAssembler.assemble(modelDir.toPath()); + } catch (IllegalArgumentException e) { + throw new TerminateToolException(1, e.getMessage()); + } catch (IOException e) { + throw new TerminateToolException(-1, + "IO error while assembling " + modelDir + ": " + e.getMessage(), e); + } + if (result.wroteVocabulary()) { + System.out.println("Wrote vocab.txt derived from tokenizer.json"); + } + if (result.wroteTokenizerConfig()) { + System.out.println("Wrote tokenizer_config.json derived from tokenizer.json"); + } + System.out.println("Assembled and verified a " + result.family() + " model: " + + result.vocabularySize() + " rows, dimension " + result.dimension()); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java new file mode 100644 index 0000000000..c4a9bf6d75 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opennlp.embeddings.cmdline; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import opennlp.tools.cmdline.BasicCmdLineTool; +import opennlp.tools.cmdline.CmdLineTool; +import opennlp.tools.cmdline.TerminateToolException; +import opennlp.tools.util.Version; + +/** + * The command line dispatcher for the OpenNLP static embeddings tools. + */ +public final class CLI { + + private static final Logger logger = LoggerFactory.getLogger(CLI.class); + static final String CMD = "opennlp-embeddings"; + + private static Map toolLookupMap; + + static { + toolLookupMap = new LinkedHashMap<>(); + + final List tools = new LinkedList<>(); + + tools.add(new AssembleModelTool()); + + for (CmdLineTool tool : tools) { + toolLookupMap.put(tool.getName(), tool); + } + + toolLookupMap = Collections.unmodifiableMap(toolLookupMap); + } + + private CLI() { + } + + /** + * @return A set which contains all tool names. + */ + public static Set getToolNames() { + return toolLookupMap.keySet(); + } + + private static void usage() { + logger.info("OpenNLP Static Embeddings {}.", Version.currentVersion()); + logger.info("Usage: {} TOOL", CMD); + + // distance of tool name from line start + int numberOfSpaces = -1; + for (String toolName : toolLookupMap.keySet()) { + if (toolName.length() > numberOfSpaces) { + numberOfSpaces = toolName.length(); + } + } + numberOfSpaces = numberOfSpaces + 4; + + final StringBuilder sb = new StringBuilder("where TOOL is one of: \n\n"); + for (CmdLineTool tool : toolLookupMap.values()) { + + sb.append(" ").append(tool.getName()); + sb.append(" ".repeat(Math.max(0, StrictMath.abs( + tool.getName().length() - numberOfSpaces)))); + sb.append(tool.getShortDescription()).append("\n"); + } + logger.info(sb.toString()); + + logger.info("All tools print help when invoked with help parameter"); + logger.info("Example: {} AssembleModel help", CMD); + } + + public static void main(String[] args) { + + if (args.length == 0) { + usage(); + System.exit(0); + } + + final String[] toolArguments = new String[args.length - 1]; + System.arraycopy(args, 1, toolArguments, 0, toolArguments.length); + + final String toolName = args[0]; + + final CmdLineTool tool = toolLookupMap.get(toolName); + + try { + if (null == tool) { + throw new TerminateToolException(1, "Tool " + toolName + " is not found."); + } + + if ((0 == toolArguments.length && tool.hasParams()) + || 0 < toolArguments.length && "help".equals(toolArguments[0])) { + logger.info(tool.getHelp()); + System.exit(0); + } + + if (tool instanceof BasicCmdLineTool basicTool) { + basicTool.run(toolArguments); + } else { + throw new TerminateToolException(1, "Tool " + toolName + " is not supported."); + } + } catch (TerminateToolException e) { + logger.error(e.getLocalizedMessage(), e); + System.exit(e.getCode()); + } + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java new file mode 100644 index 0000000000..f4a676e724 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.embeddings.cmdline.AssembleModelTool; +import opennlp.tools.cmdline.TerminateToolException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The assembler completes a distilled directory into a loadable one: it derives the WordPiece + * {@code vocab.txt} and {@code tokenizer_config.json} from {@code tokenizer.json}, leaves existing + * files alone, and reports the SentencePiece {@code .model} it cannot fabricate. The CLI tool wraps + * it and turns failures into a {@link TerminateToolException}. + */ +class ModelAssemblerTest { + + // A WordPiece tokenizer.json with a five-entry vocab dictionary (no [CLS]/[SEP], as Model2Vec + // ships) and a BERT normalizer that lower-cases. The dictionary is written out of id order to + // prove the assembler sorts it. + private static final String WORDPIECE_TOKENIZER_JSON = + "{\"version\":\"1.0\"," + + "\"normalizer\":{\"type\":\"BertNormalizer\",\"strip_accents\":null," + + "\"lowercase\":true}," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[PAD]\":0,\"hello\":2,\"[UNK]\":1,\"cat\":4,\"world\":3}}}"; + + private static final float[][] ROWS = { + {0f, 0f, 0f}, // [PAD] + {1f, 10f, 100f}, // [UNK] + {2f, 20f, 200f}, // hello + {3f, 30f, 300f}, // world + {4f, 40f, 400f}, // cat + }; + + private static Path writeWordpieceDistillation(Path dir) throws IOException { + Files.writeString(dir.resolve("tokenizer.json"), WORDPIECE_TOKENIZER_JSON); + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false}"); + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", ROWS)); + return dir; + } + + @Test + void testDerivesTheWordpieceVocabularyAndConfigInIdOrder(@TempDir Path dir) throws IOException { + writeWordpieceDistillation(dir); + + final ModelAssembler.Result result = ModelAssembler.assemble(dir); + + assertEquals("WordPiece", result.family()); + assertEquals(3, result.dimension()); + assertEquals(5, result.vocabularySize()); + assertTrue(result.wroteVocabulary()); + assertTrue(result.wroteTokenizerConfig()); + // The vocab.txt must be the dictionary in id order, not the order it was written. + assertEquals(List.of("[PAD]", "[UNK]", "hello", "world", "cat"), + Files.readAllLines(dir.resolve("vocab.txt"))); + // The casing comes from the BERT normalizer's lowercase flag. + assertTrue(Files.readString(dir.resolve("tokenizer_config.json")).contains("\"do_lower_case\": true")); + } + + @Test + void testAssembledDirectoryEmbeds(@TempDir Path dir) throws IOException { + writeWordpieceDistillation(dir); + ModelAssembler.assemble(dir); + + final StaticEmbeddingModel model = StaticEmbeddingModel.load(dir); + // (hello[row 2] + world[row 3]) / 2 = (2 + 3) / 2 in the first component; the model has no + // frame tokens, so only the two content pieces pool. + assertEquals(2.5f, model.embed("hello world")[0], 1e-5f); + } + + @Test + void testLeavesExistingFilesUntouched(@TempDir Path dir) throws IOException { + writeWordpieceDistillation(dir); + // A vocab.txt the caller already wrote must not be overwritten. + Files.write(dir.resolve("vocab.txt"), List.of("[PAD]", "[UNK]", "hello", "world", "cat")); + Files.writeString(dir.resolve("tokenizer_config.json"), "{\"do_lower_case\": false}"); + + final ModelAssembler.Result result = ModelAssembler.assemble(dir); + + assertFalse(result.wroteVocabulary()); + assertFalse(result.wroteTokenizerConfig()); + assertTrue(Files.readString(dir.resolve("tokenizer_config.json")).contains("false")); + } + + @Test + void testRejectsAMissingDistillationFile(@TempDir Path dir) throws IOException { + Files.writeString(dir.resolve("tokenizer.json"), WORDPIECE_TOKENIZER_JSON); + // no model.safetensors, no config.json + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> ModelAssembler.assemble(dir)); + assertTrue(e.getMessage().contains("model.safetensors"), e.getMessage()); + } + + @Test + void testReportsTheMissingSentencePieceModelFile(@TempDir Path dir) throws IOException { + // A Unigram distillation without its trained .model file: the assembler cannot fabricate it. + Files.writeString(dir.resolve("tokenizer.json"), + "{\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"\",0.0],[\"a\",-1.0]]}}"); + Files.writeString(dir.resolve("config.json"), "{\"normalize\":true}"); + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", new float[][] {{0f, 0f}, {1f, 1f}})); + + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> ModelAssembler.assemble(dir)); + assertTrue(e.getMessage().contains("sentencepiece.bpe.model"), e.getMessage()); + assertTrue(e.getMessage().contains("copy it from the teacher"), e.getMessage()); + } + + @Test + void testLoadsTheRealSentencePieceModelAfterItsFileIsPresent(@TempDir Path dir) + throws IOException { + // Assemble a SentencePiece directory around the bundled tiny model: once its .model file is + // present the assembler only has to verify it loads. + final byte[] modelBytes; + try (InputStream in = getClass().getResourceAsStream("/opennlp/embeddings/tiny-unigram.model")) { + modelBytes = in.readAllBytes(); + } + Files.write(dir.resolve("sentencepiece.bpe.model"), modelBytes); + Files.writeString(dir.resolve("config.json"), "{\"normalize\":false}"); + // A tokenizer.json whose vocab is the model's own poolable pieces, so the coverage check + // passes; the matrix carries one row per piece. + final opennlp.subword.sentencepiece.SentencePieceTokenizer tokenizer = + opennlp.subword.sentencepiece.SentencePieceTokenizer.load(dir.resolve("sentencepiece.bpe.model")); + final StringBuilder vocab = new StringBuilder("{\"model\":{\"type\":\"Unigram\",\"vocab\":["); + int rows = 0; + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + if (rows > 0) { + vocab.append(','); + } + vocab.append('[').append(jsonString(tokenizer.idToPiece(id))).append(",-1.0]"); + rows++; + } + vocab.append("]}}"); + Files.writeString(dir.resolve("tokenizer.json"), vocab.toString()); + final float[][] matrix = new float[rows][2]; + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", matrix)); + + final ModelAssembler.Result result = ModelAssembler.assemble(dir); + assertEquals("SentencePiece", result.family()); + assertEquals(rows, result.vocabularySize()); + assertFalse(result.wroteVocabulary()); + } + + @Test + void testToolPrintsASummaryAndFailsLoudlyOnABadDirectory(@TempDir Path dir) throws IOException { + writeWordpieceDistillation(dir); + // The tool runs the assembly without throwing on a good directory. + new AssembleModelTool().run(new String[] {"-modelDir", dir.toString()}); + + // A directory that is not a model fails as a TerminateToolException, not a raw exception. + final Path empty = Files.createDirectory(dir.resolve("empty")); + final TerminateToolException e = assertThrows(TerminateToolException.class, + () -> new AssembleModelTool().run(new String[] {"-modelDir", empty.toString()})); + assertTrue(e.getMessage().contains("tokenizer.json") || e.getMessage().contains("distilled"), + e.getMessage()); + } + + private static String jsonString(String s) { + final StringBuilder out = new StringBuilder("\""); + for (int i = 0; i < s.length(); i++) { + final char c = s.charAt(i); + switch (c) { + case '"' -> out.append("\\\""); + case '\\' -> out.append("\\\\"); + default -> { + if (c < 0x20) { + out.append(String.format("\\u%04x", (int) c)); + } else { + out.append(c); + } + } + } + } + return out.append('"').toString(); + } +} From 8b2c075e0902c4085d23ec8b90614a24801a4c33 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 13 Jul 2026 01:09:43 -0400 Subject: [PATCH 49/82] OPENNLP-1877: Ship the distillation script and the Python parity harness scripts/distill_bge_m3.py is the runnable form of the TRAINING.md worked example. scripts/parity holds the reproducible comparison against the model2vec Python reference: the same model and the same multilingual sentences on both sides, the two vector sets checked against each other, and both single-thread throughputs measured with the same fixed-duration methodology. A run passes only when the vectors agree within float tolerance, so the two speeds it prints are for implementations producing the same answer. --- .../opennlp-embeddings/TRAINING.md | 2 + .../opennlp-embeddings/scripts/README.md | 38 ++++++++ .../scripts/distill_bge_m3.py | 52 +++++++++++ .../scripts/parity/EmbedBenchM3.java | 92 ++++++++++++++++++ .../scripts/parity/parity_speed.py | 93 +++++++++++++++++++ .../opennlp-embeddings/scripts/parity/run.sh | 46 +++++++++ .../scripts/parity/sentences.txt | 20 ++++ rat-excludes | 2 + 8 files changed, 345 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/scripts/README.md create mode 100644 opennlp-extensions/opennlp-embeddings/scripts/distill_bge_m3.py create mode 100644 opennlp-extensions/opennlp-embeddings/scripts/parity/EmbedBenchM3.java create mode 100644 opennlp-extensions/opennlp-embeddings/scripts/parity/parity_speed.py create mode 100755 opennlp-extensions/opennlp-embeddings/scripts/parity/run.sh create mode 100644 opennlp-extensions/opennlp-embeddings/scripts/parity/sentences.txt diff --git a/opennlp-extensions/opennlp-embeddings/TRAINING.md b/opennlp-extensions/opennlp-embeddings/TRAINING.md index 450032f5ed..c229c3d771 100644 --- a/opennlp-extensions/opennlp-embeddings/TRAINING.md +++ b/opennlp-extensions/opennlp-embeddings/TRAINING.md @@ -45,6 +45,8 @@ print("dim:", static.dim) .venv-distill/bin/python distill_bge_m3.py ``` +This exact script ships in the module as `scripts/distill_bge_m3.py`, and `scripts/parity/` holds a harness that reruns the parity check and the single-thread speed comparison against the Python reference on any machine. + ### On the dimension `pca_dims` is the one quality knob worth thinking about, and bigger is not better. Distilling bge-m3 at 256 and at 512 gives the same cross-lingual similarity within noise (English/Chinese paraphrase around 0.69 either way), while 512 doubles the matrix on disk and in memory and cuts embedding throughput. PCA to 256 already captures the useful variance of the teacher; the extra dimensions are mostly noise that dilutes the signal. 256 is a good default, and it is where the reference potion tables sit too. diff --git a/opennlp-extensions/opennlp-embeddings/scripts/README.md b/opennlp-extensions/opennlp-embeddings/scripts/README.md new file mode 100644 index 0000000000..2a488743a1 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/scripts/README.md @@ -0,0 +1,38 @@ + + +# Embeddings scripts + +Developer scripts around the static embeddings module. None of them are part of the build; they +make the module's numbers and its worked example reproducible from a checkout. + +## `distill_bge_m3.py` + +The runnable form of the TRAINING.md worked example: distills the multilingual bge-m3 teacher +into a 256-dimension static table with Model2Vec. Needs a Python environment with +`model2vec[distill]` installed; the script's header shows the setup. After it finishes, copy the +teacher's `sentencepiece.bpe.model` next to the output and verify with the `AssembleModel` +command. + +## `parity/` + +The parity and single-thread speed comparison between this module and the model2vec Python +reference: the same model and the same multilingual sentences on both sides, the two vector sets +checked against each other, and both throughputs measured with the same fixed-duration +methodology. `sh run.sh` after building the project; see the script header for the environment +overrides. A run passes only when the vectors agree within float tolerance, so the two speeds it +prints are for implementations producing the same answer. diff --git a/opennlp-extensions/opennlp-embeddings/scripts/distill_bge_m3.py b/opennlp-extensions/opennlp-embeddings/scripts/distill_bge_m3.py new file mode 100644 index 0000000000..3c59ca05b9 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/scripts/distill_bge_m3.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Distills the multilingual bge-m3 teacher into a static embedding table. + +This is the worked example from TRAINING.md as a runnable script. It needs a Python +environment with model2vec's distill extra installed: + + uv venv .venv-distill + uv pip install --python .venv-distill "model2vec[distill]" + .venv-distill/bin/python distill_bge_m3.py [output-dir] + +After it finishes, copy the teacher's trained SentencePiece file +(sentencepiece.bpe.model on the model hub) into the output directory and run the +AssembleModel command to verify the directory loads: + + opennlp-embeddings AssembleModel -modelDir + +256 dimensions is the deliberate default: distilling the same teacher at 512 gives the +same cross-lingual similarity within noise while doubling the matrix and halving embed +throughput, because PCA to 256 already captures the useful variance. +""" + +import os +import sys + +from model2vec.distill import distill + +out = sys.argv[1] if len(sys.argv) > 1 else "bge-m3-static" + +static = distill("BAAI/bge-m3", pca_dims=256) +static.save_pretrained(out) +print("SAVED:", out, "dim:", static.dim) + +print("=== output files ===") +for name in sorted(os.listdir(out)): + path = os.path.join(out, name) + print(f" {os.path.getsize(path):>12} {name}") +print("Now copy the teacher's sentencepiece.bpe.model into", out, + "and run: opennlp-embeddings AssembleModel -modelDir", out) diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/EmbedBenchM3.java b/opennlp-extensions/opennlp-embeddings/scripts/parity/EmbedBenchM3.java new file mode 100644 index 0000000000..cfa366513c --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/scripts/parity/EmbedBenchM3.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import opennlp.embeddings.StaticEmbeddingModel; + +/** + * The JVM half of the parity and speed comparison (see run.sh). Loads the static table, writes + * one vector per input sentence for the parity check, then measures single-thread embed + * throughput with the same fixed-duration, warmup-discarded methodology the Python side uses. + * + *

Args: modelDir sentencesFile vectorsOut warmupSeconds measureSeconds

+ */ +public final class EmbedBenchM3 { + + /** Not instantiable. */ + private EmbedBenchM3() { + } + + /** + * Runs the parity dump and the single-thread throughput measurement. + * + * @param args modelDir, sentencesFile, vectorsOut, warmupSeconds, measureSeconds. + * @throws Exception Thrown if a file cannot be read or written. + */ + public static void main(String[] args) throws Exception { + final Path modelDir = Path.of(args[0]); + final List sentences = Files.readAllLines(Path.of(args[1]), StandardCharsets.UTF_8) + .stream().map(String::strip).filter(s -> !s.isEmpty()).toList(); + final Path vectorsOut = Path.of(args[2]); + final int warmupSeconds = Integer.parseInt(args[3]); + final int measureSeconds = Integer.parseInt(args[4]); + + final long loadStart = System.nanoTime(); + final StaticEmbeddingModel model = StaticEmbeddingModel.load(modelDir); + final double loadMs = (System.nanoTime() - loadStart) / 1e6; + + // One vector per sentence, so the Python side can diff them for parity. + try (BufferedWriter writer = Files.newBufferedWriter(vectorsOut, StandardCharsets.UTF_8)) { + for (final String sentence : sentences) { + final float[] vector = model.embed(sentence); + final StringBuilder line = new StringBuilder(); + for (int i = 0; i < vector.length; i++) { + if (i > 0) { + line.append(' '); + } + line.append(Float.toString(vector[i])); + } + writer.write(line.toString()); + writer.newLine(); + } + } + + final long warmupEnd = System.nanoTime() + warmupSeconds * 1_000_000_000L; + int index = 0; + while (System.nanoTime() < warmupEnd) { + model.embed(sentences.get(index++ % sentences.size())); + } + + long embedded = 0; + final long measureStart = System.nanoTime(); + final long measureEnd = measureStart + measureSeconds * 1_000_000_000L; + index = 0; + while (System.nanoTime() < measureEnd) { + model.embed(sentences.get(index++ % sentences.size())); + embedded++; + } + final double seconds = (System.nanoTime() - measureStart) / 1e9; + + System.out.printf("JVM load %.0f ms | %,.0f texts/s single-thread (%d embeds in %.1fs)%n", + loadMs, embedded / seconds, embedded, seconds); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/parity_speed.py b/opennlp-extensions/opennlp-embeddings/scripts/parity/parity_speed.py new file mode 100644 index 0000000000..13a82f129c --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/scripts/parity/parity_speed.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The Python half of the parity and speed comparison, plus the final parity check. + +Loads the same static table with model2vec, writes one vector per sentence, measures +single-thread throughput with the same fixed-duration warmup-discarded loop the JVM side +uses, then loads the JVM's vectors (written first by run.sh) and reports the parity between +them. + +The point is not to declare a winner; it is to show that both implementations produce the +same vectors, and to let anyone reproduce both numbers on their own hardware. + +Usage: parity_speed.py [sentences-file] [jvm-vectors-file] +""" + +import sys +import time + +import numpy as np +from model2vec import StaticModel + +WARMUP_SECONDS = 3 +MEASURE_SECONDS = 5 + + +def read_sentences(path): + with open(path, encoding="utf-8") as handle: + return [line.strip() for line in handle if line.strip()] + + +def main(): + model_dir = sys.argv[1] + sentences_file = sys.argv[2] if len(sys.argv) > 2 else "sentences.txt" + jvm_vectors_file = sys.argv[3] if len(sys.argv) > 3 else "jvm_vectors.tsv" + sentences = read_sentences(sentences_file) + + load_start = time.time() + model = StaticModel.from_pretrained(model_dir) + load_ms = (time.time() - load_start) * 1000.0 + + python_vectors = np.array([model.encode(s) for s in sentences], dtype=np.float32) + + end = time.time() + WARMUP_SECONDS + i = 0 + while time.time() < end: + model.encode(sentences[i % len(sentences)]) + i += 1 + + embedded = 0 + i = 0 + start = time.time() + end = start + MEASURE_SECONDS + while time.time() < end: + model.encode(sentences[i % len(sentences)]) + embedded += 1 + i += 1 + seconds = time.time() - start + print(f"Python load {load_ms:.0f} ms | {embedded / seconds:,.0f} texts/s single-thread " + f"({embedded} embeds in {seconds:.1f}s)") + + jvm_vectors = np.loadtxt(jvm_vectors_file, dtype=np.float32) + if jvm_vectors.shape != python_vectors.shape: + print(f"PARITY FAIL: shape mismatch {jvm_vectors.shape} vs {python_vectors.shape}") + sys.exit(1) + + max_abs_diff = float(np.abs(python_vectors - jvm_vectors).max()) + cosines = [ + float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))) + for a, b in zip(python_vectors, jvm_vectors) + ] + print(f"Parity max abs diff {max_abs_diff:.2e} | min cosine {min(cosines):.6f} " + f"over {len(sentences)} sentences in {python_vectors.shape[1]} dims") + if min(cosines) < 0.9999: + print("PARITY FAIL: vectors diverge") + sys.exit(1) + print("Parity OK: the JVM and Python vectors are the same within float tolerance") + + +if __name__ == "__main__": + main() diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/run.sh b/opennlp-extensions/opennlp-embeddings/scripts/parity/run.sh new file mode 100755 index 0000000000..e2eaf5687e --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/scripts/parity/run.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Reproduces the parity and single-thread speed comparison between this module and the +# model2vec Python reference: the same model and the same sentences on both sides, with the +# vector sets checked against each other. Run from this directory after building the project +# (mvn install, or at least mvn compile from the repository root). +# +# Environment overrides: +# MODEL_DIR the static model directory (default: bge-m3-static in this directory; +# see ../distill_bge_m3.py and the module's TRAINING.md to produce one) +# PYTHON a Python interpreter with model2vec installed (default: python3) +set -e + +MODEL_DIR="${MODEL_DIR:-bge-m3-static}" +PYTHON="${PYTHON:-python3}" + +# The repository root is four levels above this script. +ROOT=$(cd "$(dirname "$0")/../../../.." && pwd) +CP="$ROOT/opennlp-api/target/classes:$ROOT/opennlp-core/opennlp-runtime/target/classes:$ROOT/opennlp-extensions/opennlp-subword/target/classes:$ROOT/opennlp-extensions/opennlp-embeddings/target/classes" + +echo "Model: $MODEL_DIR" +echo "Sentences: $(grep -c . sentences.txt) lines, multilingual" +echo + +# JVM side first: it writes jvm_vectors.tsv, which the Python side then diffs. +javac -cp "$CP" -d . EmbedBenchM3.java +java -cp "$CP:." EmbedBenchM3 "$MODEL_DIR" sentences.txt jvm_vectors.tsv 3 5 + +# Python side: prints its own rate, then reports parity against the JVM's vectors. +"$PYTHON" parity_speed.py "$MODEL_DIR" diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/sentences.txt b/opennlp-extensions/opennlp-embeddings/scripts/parity/sentences.txt new file mode 100644 index 0000000000..5e6b6a3370 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/scripts/parity/sentences.txt @@ -0,0 +1,20 @@ +The weather is beautiful today and the sky is clear. +Machine learning models turn text into vectors. +I would like a cup of coffee with milk please. +The quarterly financial results disappointed investors. +Das Wetter ist heute wunderschoen und der Himmel ist klar. +Maschinelles Lernen verwandelt Text in Vektoren. +Le temps est magnifique aujourd'hui et le ciel est degage. +Los modelos de aprendizaje automatico convierten texto en vectores. +今天天气很好，天空很晴朗。 +机器学习模型把文本转换成向量。 +今日はとても良い天気で空が澄んでいます。 +機械学習モデルはテキストをベクトルに変換します。 +Сегодня прекрасная погода и ясное небо. +Модели машинного обучения превращают текст в векторы. +La retrieval semantica trova documenti per significato non per parole. +Natural language processing is a field of artificial intelligence. +A quick brown fox jumps over the lazy dog near the river. +Embeddings place similar sentences close together in space. +Coffee, tea, and espresso are all popular hot drinks. +The library opens at nine in the morning on weekdays. diff --git a/rat-excludes b/rat-excludes index 0f41e6367a..e79900850e 100644 --- a/rat-excludes +++ b/rat-excludes @@ -79,3 +79,5 @@ src/test/resources/opennlp/subword/sentencepiece/*.fixtures.tsv src/test/resources/opennlp/subword/sentencepiece/corpus.txt src/test/resources/opennlp/embeddings/tiny-unigram.model + +scripts/parity/sentences.txt From dfe5834571b4c70ab283cb6b81150c9c116f796e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 13 Jul 2026 03:29:05 -0400 Subject: [PATCH 50/82] OPENNLP-1877: Trim residual commentary per review conventions --- .../embeddings/StaticEmbeddingModel.java | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 13962f4538..c5bf73ba3a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -90,9 +90,7 @@ public enum Normalization { private static final List SENTENCEPIECE_MODEL_FILE_NAMES = List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); private static final int[] NO_EXCLUDED_ROWS = new int[0]; - // Never meaningful as a "similar word" result. Includes [PAD] and [MASK], which a distilled - // table keeps although text never tokenizes to them, so they would otherwise surface as - // neighbors. + // Excluded from neighbor results, including [PAD] and [MASK] that a distilled table keeps. private static final Set WORDPIECE_SPECIAL_TOKENS = Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, WordpieceTokenizer.BERT_UNK_TOKEN, "[PAD]", "[MASK]"); @@ -104,13 +102,10 @@ public enum Normalization { private final int dimension; private final EmbeddingVocabulary vocabulary; private final SubwordTokenizer tokenizer; - // Tokenizer-id-space test for pieces that are never pooled: the WordPiece frame and unknown - // pieces, or a SentencePiece model's control and unknown pieces (whose piece string is the - // unmatched surface text, not a vocabulary entry). + // Tokenizer-id test for pieces that are never pooled (frame, control, and unknown pieces). private final IntPredicate skipPieceId; private final boolean normalize; - // Per-row L2 norms and special-token mask, precomputed at load time so the neighbor scan - // does no per-row square root or string hashing. + // Per-row L2 norms and special-token mask, precomputed at load time for the neighbor scan. private final double[] rowNorms; private final boolean[] specialRows; @@ -324,10 +319,8 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } final WordpieceEncoder tokenizer = wordpieceEncoder(vocabulary, casing == Casing.UNCASED, unknownId); - // The encoder frames every encoding with [CLS] ... [SEP], and pooling skips that frame. When - // the distillation kept the frame rows, skip them by their own ids; when it dropped them, - // wordpieceEncoder framed with the unknown id instead, so skipping the unknown id removes - // them. A negative id is the "absent" sentinel and matches no emitted piece. + // Pooling skips the [CLS]/[SEP] frame by id; an absent frame maps to the unknown id, which + // is skipped the same way. A negative id is the absent sentinel and matches no emitted piece. final int classificationId = vocabulary.id(WordpieceTokenizer.BERT_CLS_TOKEN); final int separatorId = vocabulary.id(WordpieceTokenizer.BERT_SEP_TOKEN); final IntPredicate skipPieceId = From 0349b1f0574bbf69139fe9c286a3e1394c4e13be Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 16 Jul 2026 06:17:31 -0400 Subject: [PATCH 51/82] OPENNLP-1877: Name format constants, document throws, and source or replace unsupported doc claims --- .../opennlp-embeddings/README.md | 18 +-- .../opennlp-embeddings/TRAINING.md | 6 +- .../embeddings/EmbeddingVocabulary.java | 2 + .../opennlp/embeddings/FlatJsonFields.java | 1 + .../java/opennlp/embeddings/JsonCursor.java | 2 + .../opennlp/embeddings/ModelAssembler.java | 50 +++--- .../opennlp/embeddings/ModelFileNames.java | 52 ++++++ .../opennlp/embeddings/SafetensorsFile.java | 32 ++-- .../embeddings/StaticEmbeddingModel.java | 61 ++++--- .../java/opennlp/embeddings/TensorInfo.java | 4 +- .../embeddings/TokenizerJsonVocab.java | 13 +- .../java/opennlp/embeddings/cmdline/CLI.java | 14 +- .../embeddings/SafetensorsFileTest.java | 150 ++++++++---------- .../embeddings/TokenizerJsonVocabTest.java | 7 + 14 files changed, 239 insertions(+), 173 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md index 873d575cc6..4481edf0fc 100644 --- a/opennlp-extensions/opennlp-embeddings/README.md +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -17,11 +17,9 @@ # OpenNLP Static Embeddings -Embeddings have become an essential part of AI workloads. As such, OpenNLP introduces a pure-JVM approach to embeddings with a modern Model2Vec engine. +Turn text into embedding vectors from a static (non-contextual) table: a per-token vector matrix plus subword tokenization, WordPiece or SentencePiece. It uses the same lookup-table approach as [word2vec](https://code.google.com/archive/p/word2vec/) and [GloVe](https://nlp.stanford.edu/projects/glove/). Distillation tools can compress a sentence-transformer into such a flat table (the [Model2Vec](https://github.com/MinishLab/model2vec) family is the primary target), and looking a sentence up in the table approximates the transformer's semantics at a fraction of the cost. Because SentencePiece models are supported, this includes multilingual tables distilled from encoders like the [XLM-RoBERTa](https://arxiv.org/abs/1911.02116) family. There is no model forward pass, no GPU, and no native runtime; it is pure JVM. -Turn text into embedding vectors from a static (non-contextual) table: a per-token vector matrix plus subword tokenization, WordPiece or SentencePiece. It is the modern successor to the word2vec and GloVe workflow. Distillation tools can compress a sentence-transformer into such a flat table (the Model2Vec family is the primary target), and looking a sentence up in the table approximates the transformer's semantics at a fraction of the cost. Because SentencePiece models are supported, this includes multilingual tables distilled from encoders like the XLM-RoBERTa family. There is no model forward pass, no GPU, and no native runtime; it is pure JVM. - -OpenNLP also supports ONNX models, which are inherently more accurate. Model2Vec sacrifices some accuracy for a large speed gain, and OpenNLP recognizes that trade-off, so both embedding methods are supported and share the same `TextEmbedder` seam. +OpenNLP also supports ONNX models, which are inherently more accurate. Model2Vec sacrifices some accuracy for a large speed gain, and OpenNLP recognizes that trade-off, so both embedding methods are supported and implement the same `TextEmbedder` interface. ## Quickstart @@ -60,7 +58,7 @@ flowchart LR E --> F["float[] vector"] ``` -1. **Tokenize.** The model's own subword tokenizer splits the text into pieces: WordPiece with the model's casing rule, or a trained SentencePiece model that carries its own text normalizer. Special pieces (the WordPiece `[CLS]`/`[SEP]`/`[UNK]` frame, a SentencePiece model's control and unknown pieces) never contribute to the pooled vector. +1. **Tokenize.** The model's own subword tokenizer splits the text into pieces: WordPiece with the model's casing rule, or a trained SentencePiece model that carries its own text normalizer. Special pieces (the WordPiece `[CLS]`, `[SEP]`, and `[UNK]` tokens, a SentencePiece model's control and unknown pieces) never contribute to the pooled vector. 2. **Gather.** Each piece contributes its matrix row, found by the piece *string* rather than the tokenizer's numeric id. The two files of a SentencePiece model routinely order and offset their ids differently (the fairseq convention shifts them by one, and distillation tools reorder the vocabulary outright), so string lookup is what keeps the pairing robust; a poolable piece with no matrix row fails loud at load time, not at query time. Unknown pieces are dropped, and a text with no in-vocabulary pieces embeds to a zero vector rather than raising. 3. **Weight and pool.** Per-token weights (when the model carries them) multiply into the running sum, and the sum is divided by the plain token count. This mean-pool matches the reference implementation of the targeted model family exactly, verified against it rather than assumed. 4. **Normalize.** The pooled vector is L2-normalized by default so cosine similarity is a dot product. Normalization can be turned off for models that expect raw pooled vectors. @@ -87,7 +85,7 @@ flowchart TD MAT --> M ``` -The weights are read with a purpose-built **safetensors** reader. Unlike pickle-based checkpoint formats, safetensors carries no executable content, so loading a downloaded file cannot execute arbitrary code. Tensor data streams directly into the decoded array, so the file size is not bound by Java's int-indexed arrays; a single decoded tensor is capped at the maximum Java array length (about 2.1 billion float elements), checked explicitly. +The weights are read with a purpose-built [safetensors](https://github.com/huggingface/safetensors) reader. Unlike pickle-based checkpoint formats, safetensors carries no executable content, so loading a downloaded file cannot execute arbitrary code. Tensor data streams directly into the decoded array, so the file size is not bound by Java's int-indexed arrays; a single decoded tensor is capped at the maximum Java array length (about 2.1 billion float elements), checked explicitly. ## Architecture @@ -106,11 +104,11 @@ flowchart TD DL["SentenceVectorsDL
(opennlp-dl, ONNX)"] -. implements .-> TE ``` -Two seams keep the module small. `SubwordTokenizer` is the tokenization seam: the WordPiece encoder from `opennlp-api` and the pure-JVM SentencePiece implementation from `opennlp-subword` both produce the same piece stream, so the pooling code has exactly one path. `TextEmbedder` is the embedding seam: the static path here and the contextual ONNX path in `opennlp-dl` both implement it, so callers can swap one for the other without touching their code. +Two interfaces keep the module small. `SubwordTokenizer` is the tokenization interface: the WordPiece encoder from `opennlp-api` and the pure-JVM SentencePiece implementation from `opennlp-subword` both produce the same piece stream, so the pooling code has exactly one path. `TextEmbedder` is the embedding interface: the static path here and the contextual ONNX path in `opennlp-dl` both implement it, so callers can swap one for the other without touching their code. ## Performance -A static table wins on speed and footprint because there is no model forward pass: the hot path is a vocabulary lookup, a handful of vector adds, and one normalization. The module ships a JMH benchmark (`StaticEmbeddingModelBenchmark`) that measures `embed()` and `mostSimilar()` throughput on a real model directory (`-p modelDir=/path/to/model`), so you can reproduce numbers on your own hardware and model. +A static table wins on speed and footprint because there is no model forward pass: the hot path is a vocabulary lookup, a handful of vector adds, and one normalization. The module ships a Java Microbenchmark Harness (JMH) benchmark (`StaticEmbeddingModelBenchmark`) that measures `embed()` and `mostSimilar()` throughput on a real model directory (`-p modelDir=/path/to/model`), so you can reproduce numbers on your own hardware and model. Two things drive the numbers, and the benchmark separates them. `embed()` is tokenize-and-pool, so its cost tracks the text and the tokenizer, not the table size. `mostSimilar()` is a brute-force scan over every row, so its cost tracks the vocabulary size directly. A run comparing a small WordPiece table against the large multilingual SentencePiece table makes the split visible (throughput across all cores, one machine, indicative not publishable): @@ -119,7 +117,7 @@ Two things drive the numbers, and the benchmark separates them. `embed()` is tok | potion-base-8M | WordPiece, 29.5k | ~295k ops/s | ~9,000 ops/s | | bge-m3 (distilled) | SentencePiece, 250k | ~1.47M ops/s | ~550 ops/s | -So a large multilingual vocabulary is free for embedding and expensive for a full nearest-neighbor scan; that scan is where an approximate index earns its place once the table is large. Separately, on the potion-base-8M table the JVM path ran roughly an order of magnitude faster single-threaded than the model2vec Python reference at around a fifth of the resident memory, with output vectors matching the reference within floating-point tolerance, so the speed is not bought with accuracy. Treat all of these as a starting expectation and run the benchmark on the model you plan to use. +So a large multilingual vocabulary is free for embedding and expensive for a full nearest-neighbor scan; that scan is where an approximate index earns its place once the table is large. Separately, `scripts/parity/` holds a harness that reruns the single-thread speed comparison against the Model2Vec Python reference and checks that the output vectors match it within floating-point tolerance, so a cross-runtime comparison is something you reproduce on your own hardware rather than quote. Treat all of these as a starting expectation and run the benchmark on the model you plan to use. ## Usage @@ -176,7 +174,7 @@ IntStream.range(0, docs.size()) .forEach(i -> System.out.println(docs.get(i))); ``` -Here `dot` is any dot product over two float arrays. For a full RAG-style retriever, keep the document vectors in whatever index you already use and score queries the same way. This applies to most modern search engines, since they tend to decouple the HNSW lookups from the vectors you feed them. +Here `dot` is any dot product over two float arrays. For a full retrieval-augmented generation (RAG) retriever, keep the document vectors in whatever index you already use and score queries the same way. A vector index that stores and searches precomputed vectors, such as a Hierarchical Navigable Small World (HNSW) index, does not care how those vectors were produced, so these embeddings can feed it directly. ## Getting a model diff --git a/opennlp-extensions/opennlp-embeddings/TRAINING.md b/opennlp-extensions/opennlp-embeddings/TRAINING.md index c229c3d771..e07c0a2c1a 100644 --- a/opennlp-extensions/opennlp-embeddings/TRAINING.md +++ b/opennlp-extensions/opennlp-embeddings/TRAINING.md @@ -19,7 +19,7 @@ This module loads static embedding tables; it does not produce them. A table is distilled once from a sentence-transformer teacher, offline, in Python, and then loaded in the JVM as many times as you like. This walks through distilling one and assembling the directory `StaticEmbeddingModel.load` expects, using a multilingual SentencePiece model (bge-m3) as the worked example. -The distillation tool is [Model2Vec](https://github.com/MinishLab/model2vec). It runs the teacher over its own vocabulary once, applies PCA and a Zipf weighting, and writes a flat per-token matrix. There is no training loop and no labelled data; a distillation is minutes on CPU, not hours on a GPU. +The distillation tool is [Model2Vec](https://github.com/MinishLab/model2vec). It runs the teacher over its own vocabulary once, applies principal component analysis (PCA) and a Zipf weighting (frequent tokens are down-weighted, after Zipf's law of word frequency), and writes a flat per-token matrix. There is no training loop and no labelled data; a distillation is minutes on CPU, not hours on a GPU. ## 1. Set up the distiller @@ -30,7 +30,7 @@ uv pip install --python .venv-distill "model2vec[distill]" ## 2. Distill the teacher -bge-m3 is an XLM-RoBERTa/SentencePiece model with a 250k multilingual vocabulary, native dimension 1024. +bge-m3 is an [XLM-RoBERTa](https://arxiv.org/abs/1911.02116)/SentencePiece model with a 250k multilingual vocabulary, native dimension 1024. ```python # distill_bge_m3.py @@ -106,4 +106,4 @@ A WordPiece teacher (a BERT-family model such as bge-large-en) distills the same ## Where a table's license comes from -Distillation carries the teacher's license onto the table. bge-m3 is MIT, so its distillation is freely redistributable; a table distilled from a non-commercial or share-alike teacher inherits those terms. Check the teacher before publishing a table. +Distillation carries the teacher's license onto the table. bge-m3 is published under the MIT license per its [model card](https://huggingface.co/BAAI/bge-m3) (verify at download time), so its distillation is freely redistributable; a table distilled from a non-commercial or share-alike teacher inherits those terms. Check the teacher before publishing a table. diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java index 1b63e57d19..86bd03a58c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java @@ -124,6 +124,7 @@ List orderedTokens() { * * @param token The token to look up. Must not be {@code null}. * @return The token's id, or {@code -1} when the token is not in this vocabulary. + * @throws IllegalArgumentException Thrown if {@code token} is {@code null}. */ int id(String token) { if (token == null) { @@ -143,6 +144,7 @@ int size() { * * @param id The row id. Must be within {@code [0, size())}. * @return The token at that id. + * @throws IllegalArgumentException Thrown if {@code id} is outside {@code [0, size())}. */ String token(int id) { if (id < 0 || id >= tokenById.size()) { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java index 5a6ec7a248..4dd4dc83a5 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java @@ -28,6 +28,7 @@ */ final class FlatJsonFields { + /** Not instantiable. */ private FlatJsonFields() { } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java index 9e76ee23dd..294595276d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java @@ -31,6 +31,8 @@ final class JsonCursor { private int position; /** + * Creates a cursor positioned at the start of the given JSON text. + * * @param text The JSON text to scan. Must not be {@code null}. * @param inputName What the text is (for error messages), e.g. {@code "safetensors header"} * or a file name. diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java index 06fc0997c6..b92ec01ef8 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java @@ -43,14 +43,16 @@ */ public final class ModelAssembler { - private static final String SAFETENSORS_FILE_NAME = "model.safetensors"; - private static final String TOKENIZER_JSON_FILE_NAME = "tokenizer.json"; - private static final String CONFIG_FILE_NAME = "config.json"; - private static final String VOCABULARY_FILE_NAME = "vocab.txt"; - private static final String TOKENIZER_CONFIG_FILE_NAME = "tokenizer_config.json"; - private static final List SENTENCEPIECE_MODEL_FILE_NAMES = - List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); + /** The WordPiece tokenizer family, the {@code model.type} of a BERT-style distillation. */ + private static final String FAMILY_WORDPIECE = "WordPiece"; + /** The Unigram {@code model.type} a SentencePiece distillation's {@code tokenizer.json} uses. */ + private static final String FAMILY_UNIGRAM = "Unigram"; + + /** The SentencePiece tokenizer family, reported for an assembled Unigram directory. */ + private static final String FAMILY_SENTENCEPIECE = "SentencePiece"; + + /** Not instantiable. */ private ModelAssembler() { } @@ -88,17 +90,17 @@ public static Result assemble(Path modelDirectory) throws IOException { throw new IllegalArgumentException( "Model directory does not exist or is not a directory: " + modelDirectory); } - requireFile(modelDirectory, SAFETENSORS_FILE_NAME); - requireFile(modelDirectory, CONFIG_FILE_NAME); - final Path tokenizerJson = requireFile(modelDirectory, TOKENIZER_JSON_FILE_NAME); + requireFile(modelDirectory, ModelFileNames.SAFETENSORS); + requireFile(modelDirectory, ModelFileNames.CONFIG); + final Path tokenizerJson = requireFile(modelDirectory, ModelFileNames.TOKENIZER_JSON); final TokenizerJson tokenizer = readTokenizerJson(tokenizerJson); return switch (tokenizer.modelType()) { - case "WordPiece" -> assembleWordpiece(modelDirectory, tokenizer); - case "Unigram" -> assembleSentencePiece(modelDirectory); + case FAMILY_WORDPIECE -> assembleWordpiece(modelDirectory, tokenizer); + case FAMILY_UNIGRAM -> assembleSentencePiece(modelDirectory); default -> throw new IllegalArgumentException(tokenizerJson + " has a '" - + tokenizer.modelType() + "' tokenizer model; only WordPiece and Unigram " - + "(SentencePiece) distillations are supported"); + + tokenizer.modelType() + "' tokenizer model; only " + FAMILY_WORDPIECE + " and " + + FAMILY_UNIGRAM + " (" + FAMILY_SENTENCEPIECE + ") distillations are supported"); }; } @@ -113,17 +115,17 @@ public static Result assemble(Path modelDirectory) throws IOException { */ private static Result assembleWordpiece(Path modelDirectory, TokenizerJson tokenizer) throws IOException { - final Path vocabularyFile = modelDirectory.resolve(VOCABULARY_FILE_NAME); + final Path vocabularyFile = modelDirectory.resolve(ModelFileNames.VOCABULARY); boolean wroteVocabulary = false; if (!Files.exists(vocabularyFile)) { if (tokenizer.orderedVocabulary() == null) { throw new IllegalArgumentException("tokenizer.json in " + modelDirectory - + " has no model.vocab dictionary; cannot derive " + VOCABULARY_FILE_NAME); + + " has no model.vocab dictionary; cannot derive " + ModelFileNames.VOCABULARY); } Files.write(vocabularyFile, tokenizer.orderedVocabulary()); wroteVocabulary = true; } - final Path tokenizerConfigFile = modelDirectory.resolve(TOKENIZER_CONFIG_FILE_NAME); + final Path tokenizerConfigFile = modelDirectory.resolve(ModelFileNames.TOKENIZER_CONFIG); boolean wroteTokenizerConfig = false; if (!Files.exists(tokenizerConfigFile)) { // The BERT normalizer's lowercase flag is the casing; default to lower-casing (the uncased @@ -134,7 +136,7 @@ private static Result assembleWordpiece(Path modelDirectory, TokenizerJson token wroteTokenizerConfig = true; } final StaticEmbeddingModel model = load(modelDirectory); - return new Result("WordPiece", model.dimension(), model.vocabularySize(), + return new Result(FAMILY_WORDPIECE, model.dimension(), model.vocabularySize(), wroteVocabulary, wroteTokenizerConfig); } @@ -147,14 +149,16 @@ private static Result assembleWordpiece(Path modelDirectory, TokenizerJson token * @throws IOException Thrown if loading fails to read a file. */ private static Result assembleSentencePiece(Path modelDirectory) throws IOException { - if (firstExisting(modelDirectory, SENTENCEPIECE_MODEL_FILE_NAMES) == null) { + if (firstExisting(modelDirectory, ModelFileNames.SENTENCEPIECE_MODELS) == null) { throw new IllegalArgumentException("Model directory " + modelDirectory + " is a " - + "SentencePiece model but has no trained model file (one of " - + String.join(", ", SENTENCEPIECE_MODEL_FILE_NAMES) + "); copy it from the teacher " - + "model's repository (it is named sentencepiece.bpe.model there) into this directory"); + + FAMILY_SENTENCEPIECE + " model but has no trained model file (one of " + + String.join(", ", ModelFileNames.SENTENCEPIECE_MODELS) + "); copy it from the " + + "teacher model's repository (it is named sentencepiece.bpe.model there) into this " + + "directory"); } final StaticEmbeddingModel model = load(modelDirectory); - return new Result("SentencePiece", model.dimension(), model.vocabularySize(), false, false); + return new Result(FAMILY_SENTENCEPIECE, model.dimension(), model.vocabularySize(), + false, false); } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java new file mode 100644 index 0000000000..d0d2e1d543 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.List; + +/** + * The file names of a static embedding model directory, shared by + * {@link StaticEmbeddingModel}'s loader and {@link ModelAssembler}. A WordPiece directory holds + * {@link #SAFETENSORS}, {@link #CONFIG}, {@link #VOCABULARY}, and {@link #TOKENIZER_CONFIG}; a + * SentencePiece directory holds {@link #SAFETENSORS}, {@link #CONFIG}, {@link #TOKENIZER_JSON}, + * and one of {@link #SENTENCEPIECE_MODELS}. + */ +final class ModelFileNames { + + /** The safetensors file holding the embedding matrix and optional per-token weights. */ + static final String SAFETENSORS = "model.safetensors"; + + /** The tokenizer description whose Unigram {@code model.vocab} order names the matrix rows. */ + static final String TOKENIZER_JSON = "tokenizer.json"; + + /** The model configuration carrying the {@code normalize} pooling switch. */ + static final String CONFIG = "config.json"; + + /** The BERT-style vocabulary of a WordPiece model, one token per line in row order. */ + static final String VOCABULARY = "vocab.txt"; + + /** The tokenizer configuration carrying the WordPiece {@code do_lower_case} switch. */ + static final String TOKENIZER_CONFIG = "tokenizer_config.json"; + + /** The file names SentencePiece models ship their trained {@code .model} under, in try order. */ + static final List SENTENCEPIECE_MODELS = + List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); + + /** Not instantiable. */ + private ModelFileNames() { + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 58e7e44b5a..316c71770e 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -53,6 +53,15 @@ public final class SafetensorsFile { private static final int HEADER_LENGTH_PREFIX_BYTES = 8; + /** The header's dtype marker for 32-bit IEEE floats. */ + private static final String DTYPE_F32 = "F32"; + + /** The header's dtype marker for 16-bit IEEE half floats. */ + private static final String DTYPE_F16 = "F16"; + + /** The header's dtype marker for 16-bit bfloat16 floats. */ + private static final String DTYPE_BF16 = "BF16"; + // Positional-read chunk size, a multiple of Float.BYTES so every filled chunk decodes to // whole floats. private static final int READ_CHUNK_BYTES = 1 << 20; @@ -163,7 +172,7 @@ public TensorInfo tensorInfo(String name) { /** * Decodes a floating-point tensor's data to {@code float[]}, streaming it from the file. * Accepts the {@code F32}, {@code F16} (IEEE half) and {@code BF16} (bfloat16) dtypes; the two - * 16-bit types are widened to {@code float} as they are read. {@code F16} is model2vec's + * 16-bit types are widened to {@code float} as they are read. {@code F16} is Model2Vec's * default output dtype, so this is the common case for downloaded distilled tables. * * @param name The tensor's name. Must not be {@code null}. @@ -226,9 +235,9 @@ public float[] readFloats(String name) throws IOException { */ public float[] readFloat32(String name) throws IOException { final TensorInfo info = tensorInfo(name); - if (!"F32".equals(info.dtype())) { + if (!DTYPE_F32.equals(info.dtype())) { throw new IllegalArgumentException( - "Tensor '" + name + "' has dtype " + info.dtype() + ", not F32"); + "Tensor '" + name + "' has dtype " + info.dtype() + ", not " + DTYPE_F32); } return readFloats(name); } @@ -245,14 +254,14 @@ public float[] readFloat32(String name) throws IOException { private static void decodeInto(ByteBuffer chunk, String dtype, float[] out, int offset, int count) { switch (dtype) { - case "F32" -> chunk.asFloatBuffer().get(out, offset, count); - case "F16" -> { + case DTYPE_F32 -> chunk.asFloatBuffer().get(out, offset, count); + case DTYPE_F16 -> { final ShortBuffer shorts = chunk.asShortBuffer(); for (int i = 0; i < count; i++) { out[offset + i] = Float.float16ToFloat(shorts.get()); } } - case "BF16" -> { + case DTYPE_BF16 -> { // bfloat16 is the high 16 bits of a float32: shift back up and reinterpret. final ShortBuffer shorts = chunk.asShortBuffer(); for (int i = 0; i < count; i++) { @@ -268,20 +277,21 @@ private static void decodeInto(ByteBuffer chunk, String dtype, float[] out, int * * @param dtype The tensor dtype. * @param tensorName The tensor's name, for the error message. - * @throws IllegalArgumentException if {@code dtype} is not a supported float type. + * @throws IllegalArgumentException Thrown if {@code dtype} is not a supported float type. */ private static int floatElementBytes(String dtype, String tensorName) { return switch (dtype) { - case "F32" -> Float.BYTES; - case "F16", "BF16" -> Short.BYTES; + case DTYPE_F32 -> Float.BYTES; + case DTYPE_F16, DTYPE_BF16 -> Short.BYTES; default -> throw new IllegalArgumentException("Tensor '" + tensorName + "' has dtype " - + dtype + ", not a supported float type (F32, F16, BF16)"); + + dtype + ", not a supported float type (" + DTYPE_F32 + ", " + DTYPE_F16 + ", " + + DTYPE_BF16 + ")"); }; } /** {@return whether {@code dtype} is a float type this reader decodes} */ private static boolean isFloatDtype(String dtype) { - return "F32".equals(dtype) || "F16".equals(dtype) || "BF16".equals(dtype); + return DTYPE_F32.equals(dtype) || DTYPE_F16.equals(dtype) || DTYPE_BF16.equals(dtype); } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index c5bf73ba3a..973f105c25 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -49,7 +49,7 @@ * never by tokenizer id, so the two files may order or offset their ids differently without * corrupting lookups; a piece the matrix does not carry fails loud at load time.

* - *

Special pieces (the WordPiece {@code [CLS]}/{@code [SEP]}/{@code [UNK]} frame, a + *

Special pieces (the WordPiece {@code [CLS]}, {@code [SEP]}, and {@code [UNK]} tokens, a * SentencePiece model's control and unknown pieces) are never pooled; the sum is divided by the * count of pooled pieces, not the sum of weights. A text with no in-vocabulary pieces yields a * zero vector.

@@ -81,14 +81,6 @@ public enum Normalization { private static final float NORMALIZE_EPSILON = 1e-12f; private static final String WEIGHTS_TENSOR_NAME = "weights"; - private static final String VOCABULARY_FILE_NAME = "vocab.txt"; - private static final String SAFETENSORS_FILE_NAME = "model.safetensors"; - private static final String CONFIG_FILE_NAME = "config.json"; - private static final String TOKENIZER_CONFIG_FILE_NAME = "tokenizer_config.json"; - private static final String TOKENIZER_JSON_FILE_NAME = "tokenizer.json"; - // The file names SentencePiece models ship their trained .model under, by convention family. - private static final List SENTENCEPIECE_MODEL_FILE_NAMES = - List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); private static final int[] NO_EXCLUDED_ROWS = new int[0]; // Excluded from neighbor results, including [PAD] and [MASK] that a distilled table keeps. private static final Set WORDPIECE_SPECIAL_TOKENS = @@ -102,7 +94,7 @@ public enum Normalization { private final int dimension; private final EmbeddingVocabulary vocabulary; private final SubwordTokenizer tokenizer; - // Tokenizer-id test for pieces that are never pooled (frame, control, and unknown pieces). + // Tokenizer-id test for pieces that are never pooled (delimiter, control, unknown pieces). private final IntPredicate skipPieceId; private final boolean normalize; // Per-row L2 norms and special-token mask, precomputed at load time for the neighbor scan. @@ -160,26 +152,27 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException throw new IllegalArgumentException( "Model directory does not exist or is not a directory: " + modelDirectory); } - final Path vocabularyFile = modelDirectory.resolve(VOCABULARY_FILE_NAME); + final Path vocabularyFile = modelDirectory.resolve(ModelFileNames.VOCABULARY); if (Files.isRegularFile(vocabularyFile)) { return loadWordpieceDirectory(modelDirectory, vocabularyFile); } final Path sentencePieceModelFile = firstRegularFile(modelDirectory, - SENTENCEPIECE_MODEL_FILE_NAMES); - final Path tokenizerJsonFile = modelDirectory.resolve(TOKENIZER_JSON_FILE_NAME); + ModelFileNames.SENTENCEPIECE_MODELS); + final Path tokenizerJsonFile = modelDirectory.resolve(ModelFileNames.TOKENIZER_JSON); if (sentencePieceModelFile != null && Files.isRegularFile(tokenizerJsonFile)) { return loadSentencePiece(sentencePieceModelFile, tokenizerJsonFile, - requiredFile(modelDirectory, SAFETENSORS_FILE_NAME), - requiredNormalize(requiredFile(modelDirectory, CONFIG_FILE_NAME))); + requiredFile(modelDirectory, ModelFileNames.SAFETENSORS), + requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG))); } if (Files.isRegularFile(tokenizerJsonFile)) { throw new IllegalArgumentException("Model directory " + modelDirectory + " has a " - + TOKENIZER_JSON_FILE_NAME + " but no trained SentencePiece file (" - + String.join(", ", SENTENCEPIECE_MODEL_FILE_NAMES) + "); copy the .model file " + + ModelFileNames.TOKENIZER_JSON + " but no trained SentencePiece file (" + + String.join(", ", ModelFileNames.SENTENCEPIECE_MODELS) + "); copy the .model file " + "from the model's base tokenizer next to it"); } throw new IllegalArgumentException("Model directory " + modelDirectory + " has neither a " - + VOCABULARY_FILE_NAME + " (WordPiece layout) nor a " + TOKENIZER_JSON_FILE_NAME + + ModelFileNames.VOCABULARY + " (WordPiece layout) nor a " + + ModelFileNames.TOKENIZER_JSON + " with a trained SentencePiece file (SentencePiece layout)"); } @@ -195,10 +188,11 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, Path vocabularyFile) throws IOException { - final Path safetensorsFile = requiredFile(modelDirectory, SAFETENSORS_FILE_NAME); - final Path tokenizerConfigFile = requiredFile(modelDirectory, TOKENIZER_CONFIG_FILE_NAME); + final Path safetensorsFile = requiredFile(modelDirectory, ModelFileNames.SAFETENSORS); + final Path tokenizerConfigFile = + requiredFile(modelDirectory, ModelFileNames.TOKENIZER_CONFIG); final Normalization normalization = - requiredNormalize(requiredFile(modelDirectory, CONFIG_FILE_NAME)); + requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG)); final Boolean lowerCase = FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "do_lower_case"); if (lowerCase == null) { @@ -277,8 +271,8 @@ private static Path requiredFile(Path modelDirectory, String name) { * @param vocabularyFile The {@code vocab.txt} file: one token per line, line number is the * token's row id. Must not be {@code null}, must exist, and must * contain the {@code [UNK]} token. The {@code [CLS]} and {@code [SEP]} - * frame tokens are optional: a distilled table that dropped them (as - * Model2Vec does) still loads, because the frame is never pooled. + * delimiter tokens are optional: a distilled table that dropped them + * (as Model2Vec does) still loads, because they are never pooled. * @param safetensorsFile The {@code model.safetensors} file. Must not be {@code null} and * must exist, and must contain exactly one 2-D float tensor * (the embedding matrix) whose row count matches the vocabulary size. @@ -319,8 +313,8 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } final WordpieceEncoder tokenizer = wordpieceEncoder(vocabulary, casing == Casing.UNCASED, unknownId); - // Pooling skips the [CLS]/[SEP] frame by id; an absent frame maps to the unknown id, which - // is skipped the same way. A negative id is the absent sentinel and matches no emitted piece. + // Pooling skips [CLS] and [SEP] by id; when absent they map to the unknown id, which is + // skipped the same way. A negative id is the absent sentinel and matches no emitted piece. final int classificationId = vocabulary.id(WordpieceTokenizer.BERT_CLS_TOKEN); final int separatorId = vocabulary.id(WordpieceTokenizer.BERT_SEP_TOKEN); final IntPredicate skipPieceId = @@ -332,17 +326,18 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil } /** - * Builds the WordPiece encoder, caching {@code [CLS]} and {@code [SEP]} onto the unknown row + * Builds the WordPiece encoder, mapping {@code [CLS]} and {@code [SEP]} onto the unknown row * when the distilled vocabulary dropped them. A static embedding table mean-pools its content - * pieces and never frames, so distillers routinely remove {@code [CLS]}/{@code [SEP]} from the - * table; the encoder still frames every encoding and needs an id for the frame, and pooling - * skips the frame regardless of its id, so pointing the absent frame tokens at the unknown row - * makes the model loadable without changing which pieces are pooled. + * pieces and never pools the delimiters, so distillers routinely remove + * {@code [CLS]}/{@code [SEP]} from the table; the encoder still wraps every encoding in them + * and needs an id for each, and pooling skips them regardless of their ids, so pointing the + * absent delimiter tokens at the unknown row makes the model loadable without changing which + * pieces are pooled. * * @param vocabulary The matrix row vocabulary; must contain the unknown token. * @param lowerCase Whether the tokenizer lower-cases and strips accents. - * @param unknownId The unknown token's row, reused as the frame id when a frame token is - * absent. + * @param unknownId The unknown token's row, reused as the id of {@code [CLS]} or + * {@code [SEP]} when that token is absent. * @return The encoder. */ private static WordpieceEncoder wordpieceEncoder(EmbeddingVocabulary vocabulary, @@ -845,6 +840,8 @@ private static final class TopK { private int size; /** + * Creates an empty selection. + * * @param capacity The maximum number of rows to keep. */ TopK(int capacity) { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java index 3234a2aa36..73b3993b1e 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java @@ -56,8 +56,8 @@ public record TensorInfo(String name, String dtype, int[] shape, long dataOffset } /** - * @return The tensor's dimensions, outermost first, as a copy; mutating it does not affect - * this record. + * {@return the tensor's dimensions, outermost first, as a copy; mutating it does not affect + * this record} */ @Override public int[] shape() { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java index c4e5959204..6f5d96d5da 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java @@ -35,6 +35,7 @@ */ final class TokenizerJsonVocab { + /** Not instantiable. */ private TokenizerJsonVocab() { } @@ -52,12 +53,18 @@ private record AddedToken(long id, String content) { * * @param file The {@code tokenizer.json} file. Must not be {@code null} and must exist. * @return The pieces; the index is the matrix row. - * @throws IllegalArgumentException Thrown if the file is not a well-formed - * {@code tokenizer.json}, its model is not Unigram, or an added token's id neither matches - * an existing row nor appends as the next one. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing, the + * file is not a well-formed {@code tokenizer.json}, its model is not Unigram, or an added + * token's id neither matches an existing row nor appends as the next one. * @throws IOException Thrown if reading the file fails. */ static List rows(Path file) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); + } final String json = Files.readString(file); final JsonCursor cursor = new JsonCursor(json, file.getFileName().toString()); cursor.skipWhitespace(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java index c4a9bf6d75..5fd3a332d8 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java @@ -56,16 +56,16 @@ public final class CLI { toolLookupMap = Collections.unmodifiableMap(toolLookupMap); } + /** Not instantiable. */ private CLI() { } - /** - * @return A set which contains all tool names. - */ + /** {@return the names of all tools this command line dispatcher can run} */ public static Set getToolNames() { return toolLookupMap.keySet(); } + /** Logs the version banner and the list of available tools with their short descriptions. */ private static void usage() { logger.info("OpenNLP Static Embeddings {}.", Version.currentVersion()); logger.info("Usage: {} TOOL", CMD); @@ -93,6 +93,14 @@ private static void usage() { logger.info("Example: {} AssembleModel help", CMD); } + /** + * Runs the tool named by the first argument, passing it the remaining arguments. Without + * arguments it logs the usage overview instead, and a tool invoked with the {@code help} + * parameter logs that tool's help. Exits the JVM with the tool's error code when the tool + * terminates exceptionally. + * + * @param args The tool name followed by that tool's arguments; may be empty. + */ public static void main(String[] args) { if (args.length == 0) { diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java index 46b6226457..07bcf31542 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java @@ -23,10 +23,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Set; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,9 +38,19 @@ class SafetensorsFileTest { - // Builds a well-formed safetensors file: an 8-byte little-endian header length, the header - // JSON verbatim, then the raw data bytes. The header's data_offsets are expected to already - // be correct for the given data layout; callers construct both together. + private static final String MODEL_FILE_NAME = "model.safetensors"; + + // Builds the header JSON of a file holding one tensor, for tests that hand-roll headers with + // deliberately odd dtypes, shapes, or data_offsets. + private static String singleTensorHeader(String name, String dtype, String shape, + long begin, long end) { + return "{\"" + name + "\":{\"dtype\":\"" + dtype + "\",\"shape\":" + shape + + ",\"data_offsets\":[" + begin + "," + end + "]}}"; + } + + // Builds a safetensors file byte for byte: an 8-byte little-endian header length, the header + // JSON verbatim, then the raw data bytes. Used by the negative tests whose headers + // SafetensorsTestFiles would refuse to write; well-formed fixtures use that helper instead. private static Path writeFile(Path dir, String name, String headerJson, byte[] data) throws IOException { final byte[] headerBytes = headerJson.getBytes(StandardCharsets.UTF_8); @@ -61,11 +74,9 @@ private static byte[] floatsToLittleEndianBytes(float... values) { @Test void testRoundTripsAFloat32Matrix(@TempDir Path dir) throws IOException { - final float[] values = {1f, 2f, 3f, 4f, 5f, 6f}; - final byte[] data = floatsToLittleEndianBytes(values); - final String header = "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2,3]," - + "\"data_offsets\":[0," + data.length + "]}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final Path file = dir.resolve(MODEL_FILE_NAME); + SafetensorsTestFiles.write(file, + SafetensorsTestFiles.matrix("weight", new float[][] {{1f, 2f, 3f}, {4f, 5f, 6f}})); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -75,25 +86,19 @@ void testRoundTripsAFloat32Matrix(@TempDir Path dir) throws IOException { assertEquals("F32", info.dtype()); assertArrayEquals(new int[] {2, 3}, info.shape()); assertEquals(6, info.elementCount()); - assertArrayEquals(values, parsed.readFloat32("weight")); + assertArrayEquals(new float[] {1f, 2f, 3f, 4f, 5f, 6f}, parsed.readFloat32("weight")); } @Test void testMultipleTensorsPreserveHeaderOrder(@TempDir Path dir) throws IOException { - final byte[] a = floatsToLittleEndianBytes(1f, 2f); - final byte[] b = floatsToLittleEndianBytes(3f, 4f, 5f); - final String header = "{\"first\":{\"dtype\":\"F32\",\"shape\":[2]," - + "\"data_offsets\":[0," + a.length + "]}," - + "\"second\":{\"dtype\":\"F32\",\"shape\":[3]," - + "\"data_offsets\":[" + a.length + "," + (a.length + b.length) + "]}}"; - final ByteArrayOutputStream data = new ByteArrayOutputStream(); - data.write(a); - data.write(b); - final Path file = writeFile(dir, "model.safetensors", header, data.toByteArray()); + final Path file = dir.resolve(MODEL_FILE_NAME); + SafetensorsTestFiles.write(file, + SafetensorsTestFiles.vector("first", new float[] {1f, 2f}), + SafetensorsTestFiles.vector("second", new float[] {3f, 4f, 5f})); final SafetensorsFile parsed = SafetensorsFile.read(file); - assertEquals(java.util.List.of("first", "second"), java.util.List.copyOf(parsed.tensorNames())); + assertEquals(List.of("first", "second"), List.copyOf(parsed.tensorNames())); assertArrayEquals(new float[] {1f, 2f}, parsed.readFloat32("first")); assertArrayEquals(new float[] {3f, 4f, 5f}, parsed.readFloat32("second")); } @@ -103,7 +108,7 @@ void testMetadataMapIsParsed(@TempDir Path dir) throws IOException { final byte[] data = floatsToLittleEndianBytes(1f); final String header = "{\"__metadata__\":{\"format\":\"pt\",\"note\":\"line\\nbreak\"}," + "\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0," + data.length + "]}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, data); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -117,7 +122,7 @@ void testUnknownHeaderFieldsAreSkipped(@TempDir Path dir) throws IOException { final byte[] data = floatsToLittleEndianBytes(1f, 2f); final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2]," + "\"data_offsets\":[0," + data.length + "],\"future_field\":{\"nested\":[1,2,3]}}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, data); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -126,16 +131,10 @@ void testUnknownHeaderFieldsAreSkipped(@TempDir Path dir) throws IOException { @Test void testSingleMatrixTensorNameFindsTheOnly2DFloat32Tensor(@TempDir Path dir) throws IOException { - final byte[] scalar = floatsToLittleEndianBytes(9f); - final byte[] matrix = floatsToLittleEndianBytes(1f, 2f, 3f, 4f); - final String header = "{\"bias\":{\"dtype\":\"F32\",\"shape\":[1]," - + "\"data_offsets\":[0," + scalar.length + "]}," - + "\"embeddings\":{\"dtype\":\"F32\",\"shape\":[2,2]," - + "\"data_offsets\":[" + scalar.length + "," + (scalar.length + matrix.length) + "]}}"; - final ByteArrayOutputStream data = new ByteArrayOutputStream(); - data.write(scalar); - data.write(matrix); - final Path file = writeFile(dir, "model.safetensors", header, data.toByteArray()); + final Path file = dir.resolve(MODEL_FILE_NAME); + SafetensorsTestFiles.write(file, + SafetensorsTestFiles.vector("bias", new float[] {9f}), + SafetensorsTestFiles.matrix("embeddings", new float[][] {{1f, 2f}, {3f, 4f}})); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -144,16 +143,10 @@ void testSingleMatrixTensorNameFindsTheOnly2DFloat32Tensor(@TempDir Path dir) th @Test void testSingleMatrixTensorNameRejectsAmbiguity(@TempDir Path dir) throws IOException { - final byte[] a = floatsToLittleEndianBytes(1f, 2f, 3f, 4f); - final byte[] b = floatsToLittleEndianBytes(5f, 6f, 7f, 8f); - final String header = "{\"a\":{\"dtype\":\"F32\",\"shape\":[2,2]," - + "\"data_offsets\":[0," + a.length + "]}," - + "\"b\":{\"dtype\":\"F32\",\"shape\":[2,2]," - + "\"data_offsets\":[" + a.length + "," + (a.length + b.length) + "]}}"; - final ByteArrayOutputStream data = new ByteArrayOutputStream(); - data.write(a); - data.write(b); - final Path file = writeFile(dir, "model.safetensors", header, data.toByteArray()); + final Path file = dir.resolve(MODEL_FILE_NAME); + SafetensorsTestFiles.write(file, + SafetensorsTestFiles.matrix("a", new float[][] {{1f, 2f}, {3f, 4f}}), + SafetensorsTestFiles.matrix("b", new float[][] {{5f, 6f}, {7f, 8f}})); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -162,10 +155,8 @@ void testSingleMatrixTensorNameRejectsAmbiguity(@TempDir Path dir) throws IOExce @Test void testSingleMatrixTensorNameRejectsNoCandidate(@TempDir Path dir) throws IOException { - final byte[] data = floatsToLittleEndianBytes(1f); - final String header = "{\"bias\":{\"dtype\":\"F32\",\"shape\":[1]," - + "\"data_offsets\":[0," + data.length + "]}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final Path file = dir.resolve(MODEL_FILE_NAME); + SafetensorsTestFiles.write(file, SafetensorsTestFiles.vector("bias", new float[] {1f})); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -175,9 +166,8 @@ void testSingleMatrixTensorNameRejectsNoCandidate(@TempDir Path dir) throws IOEx @Test void testReadFloat32RejectsWrongDtype(@TempDir Path dir) throws IOException { final byte[] data = new byte[] {1, 2}; - final String header = "{\"ids\":{\"dtype\":\"I64\",\"shape\":[1]," - + "\"data_offsets\":[0,2]}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final String header = singleTensorHeader("ids", "I64", "[1]", 0, 2); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, data); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -189,8 +179,8 @@ void testReadFloat32RejectsWrongDtype(@TempDir Path dir) throws IOException { @Test void testTensorInfoRejectsUnknownName(@TempDir Path dir) throws IOException { final byte[] data = floatsToLittleEndianBytes(1f); - final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final String header = singleTensorHeader("w", "F32", "[1]", 0, 4); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, data); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -228,7 +218,7 @@ void testRejectsDuplicateTensorName(@TempDir Path dir) throws IOException { // header parser itself does not reject it; SafetensorsFile's post-parse check does. final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}," + "\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; - final Path file = writeFile(dir, "model.safetensors", header, new byte[] {1, 2, 3, 4}); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[] {1, 2, 3, 4}); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); @@ -238,15 +228,15 @@ void testRejectsDuplicateTensorName(@TempDir Path dir) throws IOException { @Test void testRejectsTensorMissingRequiredField(@TempDir Path dir) throws IOException { final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]}}"; - final Path file = writeFile(dir, "model.safetensors", header, new byte[0]); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[0]); assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); } @Test void testRejectsDataOffsetsOutOfRange(@TempDir Path dir) throws IOException { - final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,999]}}"; - final Path file = writeFile(dir, "model.safetensors", header, new byte[] {1, 2, 3, 4}); + final String header = singleTensorHeader("w", "F32", "[1]", 0, 999); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[] {1, 2, 3, 4}); assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); } @@ -254,7 +244,7 @@ void testRejectsDataOffsetsOutOfRange(@TempDir Path dir) throws IOException { @Test void testRejectsUnterminatedString(@TempDir Path dir) throws IOException { final String header = "{\"w\":{\"dtype\":\"F32"; - final Path file = writeFile(dir, "model.safetensors", header, new byte[0]); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[0]); assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); } @@ -264,9 +254,8 @@ void testRejectsTensorLargerThanAJavaArray(@TempDir Path dir) throws IOException // 2_000_000 * 2_000 = 4 billion elements, over the float[] ceiling. The bogus small data // range keeps the file tiny; the array-ceiling check fires before the range-mismatch check // because it subsumes it for tensors this large. - final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2000000,2000]," - + "\"data_offsets\":[0,4]}}"; - final Path file = writeFile(dir, "model.safetensors", header, new byte[] {1, 2, 3, 4}); + final String header = singleTensorHeader("w", "F32", "[2000000,2000]", 0, 4); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[] {1, 2, 3, 4}); final SafetensorsFile parsed = SafetensorsFile.read(file); @@ -280,12 +269,11 @@ void testFailsLoudWhenTheFileIsTruncatedAfterRead(@TempDir Path dir) throws IOEx // Tensor data is streamed on demand rather than held in memory, so a file that shrinks // between read() and readFloat32() must fail loud, not return partial data. final byte[] data = floatsToLittleEndianBytes(1f, 2f); - final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2]," - + "\"data_offsets\":[0," + data.length + "]}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final String header = singleTensorHeader("w", "F32", "[2]", 0, data.length); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, data); final SafetensorsFile parsed = SafetensorsFile.read(file); - writeFile(dir, "model.safetensors", header, floatsToLittleEndianBytes(1f)); + writeFile(dir, MODEL_FILE_NAME, header, floatsToLittleEndianBytes(1f)); final IllegalStateException e = assertThrows(IllegalStateException.class, () -> parsed.readFloat32("w")); @@ -296,9 +284,8 @@ void testFailsLoudWhenTheFileIsTruncatedAfterRead(@TempDir Path dir) throws IOEx void testReadFloat32RejectsElementCountByteRangeMismatch(@TempDir Path dir) throws IOException { // Shape [2] declares two F32 elements (8 bytes) but the data range holds only one. final byte[] data = floatsToLittleEndianBytes(1f); - final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[2]," - + "\"data_offsets\":[0," + data.length + "]}}"; - final Path file = writeFile(dir, "model.safetensors", header, data); + final String header = singleTensorHeader("w", "F32", "[2]", 0, data.length); + final Path file = writeFile(dir, MODEL_FILE_NAME, header, data); final SafetensorsFile parsed = SafetensorsFile.read(file); final IllegalArgumentException e = @@ -334,32 +321,23 @@ void testTensorInfoElementCountOverflowFailsLoudly() { assertTrue(e.getMessage().contains("overflows"), e.getMessage()); } - @Test - void testReadsF16TensorWidenedToFloat(@TempDir Path dir) throws IOException { - // F16 is model2vec's default output dtype, so this is the common downloaded-model case. - final Path file = dir.resolve("f16.safetensors"); - final float[] expected = {1.0f, -2.0f, 0.5f, 3.5f}; // all exact in IEEE half - SafetensorsTestFiles.write(file, "F16", SafetensorsTestFiles.vector("w", expected)); - - final SafetensorsFile parsed = SafetensorsFile.read(file); - assertEquals("F16", parsed.tensorInfo("w").dtype()); - assertArrayEquals(expected, parsed.readFloats("w"), 1e-3f); - } - - @Test - void testReadsBf16TensorWidenedToFloat(@TempDir Path dir) throws IOException { - final Path file = dir.resolve("bf16.safetensors"); - final float[] expected = {1.0f, -2.0f, 0.5f, 100.0f}; // exact in bfloat16 - SafetensorsTestFiles.write(file, "BF16", SafetensorsTestFiles.vector("w", expected)); + // F16 is Model2Vec's default output dtype, so widening is the common downloaded-model case; + // BF16 takes the same path with a different bit layout. + @ParameterizedTest + @ValueSource(strings = {"F16", "BF16"}) + void testReads16BitTensorWidenedToFloat(String dtype, @TempDir Path dir) throws IOException { + final Path file = dir.resolve(MODEL_FILE_NAME); + final float[] expected = {1.0f, -2.0f, 0.5f, 3.5f}; // exact in both 16-bit formats + SafetensorsTestFiles.write(file, dtype, SafetensorsTestFiles.vector("w", expected)); final SafetensorsFile parsed = SafetensorsFile.read(file); - assertEquals("BF16", parsed.tensorInfo("w").dtype()); + assertEquals(dtype, parsed.tensorInfo("w").dtype()); assertArrayEquals(expected, parsed.readFloats("w"), 1e-3f); } @Test void testSingleMatrixTensorNameAcceptsF16(@TempDir Path dir) throws IOException { - final Path file = dir.resolve("f16-matrix.safetensors"); + final Path file = dir.resolve(MODEL_FILE_NAME); SafetensorsTestFiles.write(file, "F16", SafetensorsTestFiles.matrix("embeddings", new float[][] {{1f, 2f}, {3f, 4f}})); @@ -369,7 +347,7 @@ void testSingleMatrixTensorNameAcceptsF16(@TempDir Path dir) throws IOException @Test void testReadFloat32StrictlyRejectsF16(@TempDir Path dir) throws IOException { - final Path file = dir.resolve("f16-strict.safetensors"); + final Path file = dir.resolve(MODEL_FILE_NAME); SafetensorsTestFiles.write(file, "F16", SafetensorsTestFiles.vector("w", new float[] {1f, 2f})); final SafetensorsFile parsed = SafetensorsFile.read(file); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java index 043f97a376..3e118f60d4 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java @@ -44,6 +44,13 @@ private Path write(String json) throws IOException { return file; } + @Test + void testRejectsNullAndMissingFile() { + assertThrows(IllegalArgumentException.class, () -> TokenizerJsonVocab.rows(null)); + assertThrows(IllegalArgumentException.class, + () -> TokenizerJsonVocab.rows(dir.resolve("absent.json"))); + } + @Test void testVocabListOrderIsTheRowOrder() throws IOException { final Path file = write("{\"model\":{\"type\":\"Unigram\",\"unk_id\":1," From 1fd2f680523a3d2091afdc93634e95010f4507ac Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 20 Jul 2026 04:40:23 -0400 Subject: [PATCH 52/82] OPENNLP-1877: Cite the static embedding usage example test in the manual Add StaticEmbeddingUsageExampleTest asserting the load-and-query workflow and point the embeddings manual section at it. --- opennlp-docs/src/docbkx/embeddings.xml | 4 + .../StaticEmbeddingUsageExampleTest.java | 80 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 49bb92bff2..d28b1d1b97 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -73,6 +73,10 @@ double similarity = model.similarity("coffee", "espresso"); List neighbors = model.mostSimilar("coffee", 5); List analogy = model.analogy("man", "king", "woman", 1);]]> + + StaticEmbeddingUsageExampleTest asserts the load-and-query workflow + shown here. + For a model laid out differently, the explicit overloads take the data files and the switches directly. The WordPiece overload takes whether the tokenizer lower-cases diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java new file mode 100644 index 0000000000..dfd5ca34ab --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.embeddings.StaticEmbeddingModel.Casing; +import opennlp.embeddings.StaticEmbeddingModel.Normalization; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins the cookbook path documented in {@code embeddings.xml}: load a + * {@link StaticEmbeddingModel}, embed a sentence, and call {@code similarity}, + * {@code mostSimilar}, and {@code analogy}. + */ +public class StaticEmbeddingUsageExampleTest { + + private static final List VOCAB_TOKENS = + List.of("[CLS]", "[SEP]", "[UNK]", "king", "queen", "man", "woman", "apple"); + + // king - man + woman = [3,3] - [2,1] + [1,2] = [2,4] = queen, exactly. + private static final float[][] ROWS = { + {0f, 0f}, + {0f, 0f}, + {0f, 0f}, + {3f, 3f}, + {2f, 4f}, + {2f, 1f}, + {1f, 2f}, + {-3f, -1f}, + }; + + private static StaticEmbeddingModel load(Path dir) throws IOException { + final Path vocab = dir.resolve("vocab.txt"); + Files.write(vocab, VOCAB_TOKENS); + final Path weights = dir.resolve("model.safetensors"); + SafetensorsTestFiles.write(weights, SafetensorsTestFiles.matrix("embeddings", ROWS)); + return StaticEmbeddingModel.load(vocab, weights, Casing.UNCASED, Normalization.NONE); + } + + @Test + void testEmbedSimilarityNeighborsAndAnalogy(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + final float[] vector = model.embed("king"); + assertEquals(2, vector.length); + + assertEquals(1.0, model.similarity("king", "king"), 1e-5); + + final List neighbors = model.mostSimilar("king", 5); + assertTrue(!neighbors.isEmpty()); + assertEquals("king", neighbors.get(0).token()); + + final List analogy = model.analogy("man", "king", "woman", 1); + assertEquals(1, analogy.size()); + assertEquals("queen", analogy.get(0).token()); + } +} From 9aa534c8de11c6316dec83a4fc83262570178441 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 21 Jul 2026 06:50:22 -0400 Subject: [PATCH 53/82] OPENNLP-1877: Align dev helper placement and test literals with the review conventions --- .../scripts => dev/embeddings}/README.md | 0 .../scripts => dev/embeddings}/distill_bge_m3.py | 4 ++-- .../embeddings}/parity/EmbedBenchM3.java | 0 .../scripts => dev/embeddings}/parity/parity_speed.py | 0 .../scripts => dev/embeddings}/parity/run.sh | 9 +++++---- .../scripts => dev/embeddings}/parity/sentences.txt | 0 opennlp-extensions/opennlp-embeddings/README.md | 2 +- opennlp-extensions/opennlp-embeddings/TRAINING.md | 2 +- .../StaticEmbeddingModelSentencePieceTest.java | 10 +++++----- .../opennlp/embeddings/TokenizerJsonVocabTest.java | 2 +- rat-excludes | 2 +- 11 files changed, 16 insertions(+), 15 deletions(-) rename {opennlp-extensions/opennlp-embeddings/scripts => dev/embeddings}/README.md (100%) rename {opennlp-extensions/opennlp-embeddings/scripts => dev/embeddings}/distill_bge_m3.py (92%) rename {opennlp-extensions/opennlp-embeddings/scripts => dev/embeddings}/parity/EmbedBenchM3.java (100%) rename {opennlp-extensions/opennlp-embeddings/scripts => dev/embeddings}/parity/parity_speed.py (100%) rename {opennlp-extensions/opennlp-embeddings/scripts => dev/embeddings}/parity/run.sh (88%) rename {opennlp-extensions/opennlp-embeddings/scripts => dev/embeddings}/parity/sentences.txt (100%) diff --git a/opennlp-extensions/opennlp-embeddings/scripts/README.md b/dev/embeddings/README.md similarity index 100% rename from opennlp-extensions/opennlp-embeddings/scripts/README.md rename to dev/embeddings/README.md diff --git a/opennlp-extensions/opennlp-embeddings/scripts/distill_bge_m3.py b/dev/embeddings/distill_bge_m3.py similarity index 92% rename from opennlp-extensions/opennlp-embeddings/scripts/distill_bge_m3.py rename to dev/embeddings/distill_bge_m3.py index 3c59ca05b9..5a33620e03 100644 --- a/opennlp-extensions/opennlp-embeddings/scripts/distill_bge_m3.py +++ b/dev/embeddings/distill_bge_m3.py @@ -15,8 +15,8 @@ """Distills the multilingual bge-m3 teacher into a static embedding table. -This is the worked example from TRAINING.md as a runnable script. It needs a Python -environment with model2vec's distill extra installed: +This is the worked example from opennlp-extensions/opennlp-embeddings/TRAINING.md as a +runnable script. It needs a Python environment with model2vec's distill extra installed: uv venv .venv-distill uv pip install --python .venv-distill "model2vec[distill]" diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/EmbedBenchM3.java b/dev/embeddings/parity/EmbedBenchM3.java similarity index 100% rename from opennlp-extensions/opennlp-embeddings/scripts/parity/EmbedBenchM3.java rename to dev/embeddings/parity/EmbedBenchM3.java diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/parity_speed.py b/dev/embeddings/parity/parity_speed.py similarity index 100% rename from opennlp-extensions/opennlp-embeddings/scripts/parity/parity_speed.py rename to dev/embeddings/parity/parity_speed.py diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/run.sh b/dev/embeddings/parity/run.sh similarity index 88% rename from opennlp-extensions/opennlp-embeddings/scripts/parity/run.sh rename to dev/embeddings/parity/run.sh index e2eaf5687e..6fe1e4a920 100755 --- a/opennlp-extensions/opennlp-embeddings/scripts/parity/run.sh +++ b/dev/embeddings/parity/run.sh @@ -16,22 +16,23 @@ # specific language governing permissions and limitations # under the License. -# Reproduces the parity and single-thread speed comparison between this module and the +# Reproduces the parity and single-thread speed comparison between opennlp-embeddings and the # model2vec Python reference: the same model and the same sentences on both sides, with the # vector sets checked against each other. Run from this directory after building the project # (mvn install, or at least mvn compile from the repository root). # # Environment overrides: # MODEL_DIR the static model directory (default: bge-m3-static in this directory; -# see ../distill_bge_m3.py and the module's TRAINING.md to produce one) +# see ../distill_bge_m3.py and opennlp-extensions/opennlp-embeddings/TRAINING.md +# to produce one) # PYTHON a Python interpreter with model2vec installed (default: python3) set -e MODEL_DIR="${MODEL_DIR:-bge-m3-static}" PYTHON="${PYTHON:-python3}" -# The repository root is four levels above this script. -ROOT=$(cd "$(dirname "$0")/../../../.." && pwd) +# The repository root is three levels above this script. +ROOT=$(cd "$(dirname "$0")/../../.." && pwd) CP="$ROOT/opennlp-api/target/classes:$ROOT/opennlp-core/opennlp-runtime/target/classes:$ROOT/opennlp-extensions/opennlp-subword/target/classes:$ROOT/opennlp-extensions/opennlp-embeddings/target/classes" echo "Model: $MODEL_DIR" diff --git a/opennlp-extensions/opennlp-embeddings/scripts/parity/sentences.txt b/dev/embeddings/parity/sentences.txt similarity index 100% rename from opennlp-extensions/opennlp-embeddings/scripts/parity/sentences.txt rename to dev/embeddings/parity/sentences.txt diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md index 4481edf0fc..e67036e8dc 100644 --- a/opennlp-extensions/opennlp-embeddings/README.md +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -117,7 +117,7 @@ Two things drive the numbers, and the benchmark separates them. `embed()` is tok | potion-base-8M | WordPiece, 29.5k | ~295k ops/s | ~9,000 ops/s | | bge-m3 (distilled) | SentencePiece, 250k | ~1.47M ops/s | ~550 ops/s | -So a large multilingual vocabulary is free for embedding and expensive for a full nearest-neighbor scan; that scan is where an approximate index earns its place once the table is large. Separately, `scripts/parity/` holds a harness that reruns the single-thread speed comparison against the Model2Vec Python reference and checks that the output vectors match it within floating-point tolerance, so a cross-runtime comparison is something you reproduce on your own hardware rather than quote. Treat all of these as a starting expectation and run the benchmark on the model you plan to use. +So a large multilingual vocabulary is free for embedding and expensive for a full nearest-neighbor scan; that scan is where an approximate index earns its place once the table is large. Separately, the repository's `dev/embeddings/parity/` directory holds a harness that reruns the single-thread speed comparison against the Model2Vec Python reference and checks that the output vectors match it within floating-point tolerance, so a cross-runtime comparison is something you reproduce on your own hardware rather than quote. Treat all of these as a starting expectation and run the benchmark on the model you plan to use. ## Usage diff --git a/opennlp-extensions/opennlp-embeddings/TRAINING.md b/opennlp-extensions/opennlp-embeddings/TRAINING.md index e07c0a2c1a..08f8b3eb33 100644 --- a/opennlp-extensions/opennlp-embeddings/TRAINING.md +++ b/opennlp-extensions/opennlp-embeddings/TRAINING.md @@ -45,7 +45,7 @@ print("dim:", static.dim) .venv-distill/bin/python distill_bge_m3.py ``` -This exact script ships in the module as `scripts/distill_bge_m3.py`, and `scripts/parity/` holds a harness that reruns the parity check and the single-thread speed comparison against the Python reference on any machine. +This exact script lives in the repository as `dev/embeddings/distill_bge_m3.py`, and `dev/embeddings/parity/` holds a harness that reruns the parity check and the single-thread speed comparison against the Python reference on any machine. ### On the dimension diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java index c1e53cec90..6b3ebe1842 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java @@ -160,8 +160,8 @@ private static String quote(String s) { void testEmbedGathersRowsByPieceStringAcrossTheIdOffset(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = loadFromDirectory(writeModelDirectory(dir, null)); - // "a" segments to the single piece "▁a"; the embedding must be exactly that piece's matrix - // row, found by string in the reordered vocabulary, not by the tokenizer's id. + // "a" segments to the single piece U+2581 + "a"; the embedding must be exactly that piece's + // matrix row, found by string in the reordered vocabulary, not by the tokenizer's id. final List pieces = tokenizer.encode("a"); assertEquals(1, pieces.size()); final int row = rows.indexOf(pieces.get(0).piece()); @@ -206,7 +206,7 @@ void testUnknownPiecesAreSkippedInPooling(@TempDir Path dir) throws IOException // The euro sign is outside the tiny training corpus, so it segments to the dummy-prefix // piece plus an unknown piece carrying the surface text. The unknown piece's string is not // a vocabulary entry, so pooling must skip it by its id, leaving only the mapped pieces. - final List pieces = tokenizer.encode("€"); + final List pieces = tokenizer.encode("\u20AC"); final float[] expected = new float[DIMENSION]; int pooled = 0; int unknown = 0; @@ -224,11 +224,11 @@ void testUnknownPiecesAreSkippedInPooling(@TempDir Path dir) throws IOException } pooled++; } - assertTrue(unknown > 0, "fixture assumption: '€' must produce an unknown piece"); + assertTrue(unknown > 0, "fixture assumption: the euro sign must produce an unknown piece"); for (int d = 0; d < DIMENSION; d++) { expected[d] /= Math.max(pooled, 1); } - assertArrayEquals(expected, model.embed("€"), 1e-5f); + assertArrayEquals(expected, model.embed("\u20AC"), 1e-5f); } @Test diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java index 3e118f60d4..c389e33e1c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java @@ -56,7 +56,7 @@ void testVocabListOrderIsTheRowOrder() throws IOException { final Path file = write("{\"model\":{\"type\":\"Unigram\",\"unk_id\":1," + "\"vocab\":[[\"\",0.0],[\"\",0.0],[\"\\u2581a\",-2.5],[\"b\",-3.0]]}}"); - assertEquals(List.of("", "", "▁a", "b"), TokenizerJsonVocab.rows(file)); + assertEquals(List.of("", "", "\u2581a", "b"), TokenizerJsonVocab.rows(file)); } @Test diff --git a/rat-excludes b/rat-excludes index e79900850e..90b6a69803 100644 --- a/rat-excludes +++ b/rat-excludes @@ -80,4 +80,4 @@ src/test/resources/opennlp/subword/sentencepiece/corpus.txt src/test/resources/opennlp/embeddings/tiny-unigram.model -scripts/parity/sentences.txt +dev/embeddings/parity/sentences.txt From d3cf3ff4dd3a214ce6be80cc2d1a6844b6fe5301 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 07:04:02 -0400 Subject: [PATCH 54/82] OPENNLP-1877: Add a Java model distiller and address the review comments Distillation, so producing a table no longer needs a Python environment: - Add ModelDistiller, reproducing Model2Vec in Java: clean the teacher's vocabulary, run every surviving token through the teacher's ONNX graph as [bos, token, eos] and mean-pool the last hidden states, project onto the top principal components, then scale each row by its Zipf weight sif / (sif + p). - Add OnnxTeacherEncoder for that forward pass, feeding the graph exactly the inputs it declares (input_ids, attention_mask, plus a zero token_type_ids for the BERT-family graphs that ask for one), and declare the onnxruntime dependency in the module at the root-managed version, the same engine opennlp-dl already runs. - Add TeacherTokenizer, which decides which vocabulary rows survive into the table and rewrites tokenizer.json to describe it, copying every field it does not change byte for byte so the result stays a faithful fast-tokenizer description. - Add RandomizedPca (Halko, Martinsson, Tropp), because a dense SVD of a 250k-row multilingual matrix is not practical in pure Java; component signs are fixed the way scikit-learn's svd_flip fixes them, so two runs are comparable rather than mirrored. - Add SafetensorsWriter, the write side of the format SafetensorsFile reads, streaming the matrix in chunks so its overhead beyond the caller's array is constant. - Add HuggingFaceModelCache so -teacher accepts a hub id and its files download once into a local cache; an optional file the repository does not have (a WordPiece teacher has no SentencePiece model) is reported absent, not an error. - Register the DistillModel tool in the module CLI. It ends by assembling and verifying its own output directory, so a run that prints a summary is a directory that loads. Documentation: - Rewrite TRAINING.md around the DistillModel command instead of the Python uv and model2vec setup, keep the Python flow only as the parity reference, and record that two tables distilled independently from one teacher agree on their pairwise geometry but not axis by axis. - Point the README "Getting a model" section at DistillModel as an alternative to downloading an already released table. Folded duplication: - Move firstRegularFile into ModelFileNames, next to the name lists it scans; StaticEmbeddingModel and ModelAssembler each carried a copy. - Fold the boolean and string readers of FlatJsonFields onto one top-level walker with a ValueReader seam, so the object grammar, the duplicate-field check, and the trailing-content check exist once. - Collapse the two identical file checks in EmbeddingVocabulary into requireRegularFile. - Reduce SentenceVectorsDL.declaredOutputDimension to the single first-output read the loop actually performed, since every path returned on the first output anyway. Javadoc and commentary: - Document JsonCursor.consumeLiteral, requireEnd, and the new position(), and the two writer overloads in SafetensorsTestFiles. - Fix TensorInfo's stale link to SafetensorsFile.readFloat32, which is readFloats now, and move its element-count description into the {@return} form. - Describe Neighbor.token as one subword piece of the model's tokenizer rather than a WordPiece, now that SentencePiece tables load too. - State on TextEmbedder that thread safety is implementation specific, which is what the two implementations actually promise, and mark up null in its tags. - Drop the commentary that narrates history rather than behavior: the benchmark's design-doc justification, the "before the fix" and "used to crash" notes in the similarity tests, and the "untouched by the interface adoption" note in the DL test. Tests: - Add EmbeddingTestFixtures holding the king/queen/man/woman analogy table and the JSON string quoter, and point the similarity, concurrency, SentencePiece, usage-example, and assembler tests at it instead of their own copies of each. - Build the size-mismatch and zero-vector fixtures with SafetensorsTestFiles rather than hand-rolling the header and the little-endian payload in the test. - Pin FlatJsonFields.topLevelString: escapes, absent versus explicit null, nested names not matching, non-string values, duplicates, and null arguments. - Add ModelDistillerTest, RandomizedPcaTest, and TeacherTokenizerTest over the distiller's pure pieces, and drop a duplicate IOException import in FlatJsonFieldsTest. Build: - Sort opennlp-spellcheck before opennlp-subword in the extensions module list. --- .../tools/embeddings/TextEmbedder.java | 15 +- .../opennlp/dl/vectors/SentenceVectorsDL.java | 18 +- .../SentenceVectorsDLEmbedderTest.java | 2 +- .../opennlp-embeddings/README.md | 2 +- .../opennlp-embeddings/TRAINING.md | 58 +- opennlp-extensions/opennlp-embeddings/pom.xml | 7 + .../StaticEmbeddingModelBenchmark.java | 10 +- .../embeddings/EmbeddingVocabulary.java | 20 +- .../opennlp/embeddings/FlatJsonFields.java | 87 +- .../embeddings/HuggingFaceModelCache.java | 150 +++ .../java/opennlp/embeddings/JsonCursor.java | 25 +- .../opennlp/embeddings/ModelAssembler.java | 19 +- .../opennlp/embeddings/ModelDistiller.java | 301 +++++ .../opennlp/embeddings/ModelFileNames.java | 19 + .../java/opennlp/embeddings/Neighbor.java | 4 +- .../embeddings/OnnxTeacherEncoder.java | 191 +++ .../opennlp/embeddings/RandomizedPca.java | 601 ++++++++++ .../opennlp/embeddings/SafetensorsWriter.java | 112 ++ .../embeddings/StaticEmbeddingModel.java | 25 +- .../opennlp/embeddings/TeacherTokenizer.java | 1025 +++++++++++++++++ .../java/opennlp/embeddings/TensorInfo.java | 6 +- .../java/opennlp/embeddings/cmdline/CLI.java | 1 + .../cmdline/DistillModelParams.java | 49 + .../embeddings/cmdline/DistillModelTool.java | 80 ++ .../embeddings/EmbeddingTestFixtures.java | 99 ++ .../embeddings/FlatJsonFieldsTest.java | 56 +- .../embeddings/ModelAssemblerTest.java | 27 +- .../embeddings/ModelDistillerTest.java | 73 ++ .../opennlp/embeddings/RandomizedPcaTest.java | 149 +++ .../SafetensorsHeaderParserTest.java | 1 + .../embeddings/SafetensorsTestFiles.java | 16 + .../StaticEmbeddingModelConcurrencyTest.java | 25 +- ...StaticEmbeddingModelSentencePieceTest.java | 23 +- .../StaticEmbeddingModelSimilarityTest.java | 72 +- .../embeddings/StaticEmbeddingModelTest.java | 31 +- .../StaticEmbeddingUsageExampleTest.java | 28 +- .../embeddings/TeacherTokenizerTest.java | 187 +++ 37 files changed, 3301 insertions(+), 313 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java index f1e4142090..b6b4e880f2 100644 --- a/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java +++ b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java @@ -27,10 +27,8 @@ * looks up a stored vector for a single word; an embedder composes a vector for text it has * never seen, handling tokenization and pooling internally.

* - *

Implementations are expected to be safe for concurrent use by multiple threads; any - * implementation that is not must document it. Implementation failures during encoding (a - * backing runtime error, a corrupted model) surface as unchecked exceptions carrying the - * underlying cause.

+ *

Thread safety is implementation specific. Failures during encoding surface as unchecked + * exceptions carrying the underlying cause.

*/ public interface TextEmbedder { @@ -42,9 +40,9 @@ public interface TextEmbedder { * or fallback token, or something else, and should document its choice. Callers that need a * uniform response should handle it themselves.

* - * @param text The text to embed; must not be null. + * @param text The text to embed. Must not be {@code null}. * @return The embedding vector, of length {@link #dimension()}. - * @throws IllegalArgumentException Thrown if {@code text} is null. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. */ float[] embed(CharSequence text); @@ -55,9 +53,10 @@ public interface TextEmbedder { * runtime that executes batches more efficiently than single inputs should override this * method.

* - * @param texts The texts to embed; must not be null and must not contain null. + * @param texts The texts to embed. Must not be {@code null} and must not contain {@code null}. * @return One embedding vector per input, in input order. - * @throws IllegalArgumentException Thrown if {@code texts} is null or contains null. + * @throws IllegalArgumentException Thrown if {@code texts} is {@code null} or contains + * {@code null}. */ default float[][] embedAll(List texts) { if (texts == null) { diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java index 0d41958406..8069517de3 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java @@ -22,6 +22,7 @@ import java.nio.LongBuffer; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import ai.onnxruntime.NodeInfo; @@ -149,8 +150,7 @@ public float[] getVectors(final String sentence) throws OrtException { * *

Adapts {@link #getVectors(String)} to the {@link TextEmbedder} contract. Empty or * unrecognized input is still run through the model, which returns the vector for the - * wrapped {@code [CLS] ... [SEP]} sequence; it is not special-cased to a zero vector the way - * the static-table embedder is.

+ * wrapped {@code [CLS] ... [SEP]} sequence rather than a zero vector.

* * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. * @throws IllegalStateException Thrown if inference fails; the cause carries the @@ -197,17 +197,13 @@ public int dimension() { * @throws OrtException Thrown if reading the output metadata fails. */ private static int declaredOutputDimension(final OrtSession session) throws OrtException { - for (final NodeInfo output : session.getOutputInfo().values()) { - if (output.getInfo() instanceof TensorInfo tensorInfo) { - final long[] shape = tensorInfo.getShape(); - final long last = shape.length > 0 ? shape[shape.length - 1] : -1; - if (last > 0 && last <= Integer.MAX_VALUE) { - return (int) last; - } - } + final Iterator outputs = session.getOutputInfo().values().iterator(); + if (!outputs.hasNext() || !(outputs.next().getInfo() instanceof TensorInfo tensorInfo)) { return -1; } - return -1; + final long[] shape = tensorInfo.getShape(); + final long last = shape.length > 0 ? shape[shape.length - 1] : -1; + return last > 0 && last <= Integer.MAX_VALUE ? (int) last : -1; } /** diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java index 98093d5de8..06fd847cea 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java @@ -71,7 +71,7 @@ private static File vocab(Path dir) throws IOException { void testEmbedderContractOverARealSession(@TempDir Path dir) throws Exception { try (SentenceVectorsDL vectors = new SentenceVectorsDL(model(dir), vocab(dir))) { - // The original entry point is untouched by the interface adoption. + // The primary entry point, against which the adapter below is compared. assertArrayEquals(CLS_VECTOR, vectors.getVectors("hello world"), 1e-5f); final TextEmbedder embedder = vectors; diff --git a/opennlp-extensions/opennlp-embeddings/README.md b/opennlp-extensions/opennlp-embeddings/README.md index e67036e8dc..cffd98ad3b 100644 --- a/opennlp-extensions/opennlp-embeddings/README.md +++ b/opennlp-extensions/opennlp-embeddings/README.md @@ -178,7 +178,7 @@ Here `dot` is any dot product over two float arrays. For a full retrieval-augmen ## Getting a model -No model is bundled. Point the module at files you download, and the table's own license applies to the table. The Model2Vec distilled releases (for example potion-base-8M) publish the exact directory layout the one-argument `load` expects: download that release's `vocab.txt`, `model.safetensors`, `config.json`, and `tokenizer_config.json` into one directory and pass the directory to `load`. +No model is bundled. Point the module at files you download, and the table's own license applies to the table. The Model2Vec distilled releases (for example potion-base-8M) publish the exact directory layout the one-argument `load` expects: download that release's `vocab.txt`, `model.safetensors`, `config.json`, and `tokenizer_config.json` into one directory and pass the directory to `load`. Or distill your own teacher with the module's `DistillModel` command (see `TRAINING.md`). For a multilingual SentencePiece table (for example one distilled from a bge-m3 or XLM-RoBERTa teacher), the distillation output ships `tokenizer.json`, `model.safetensors`, and `config.json` but usually not the trained SentencePiece `.model` file; copy that one file from the teacher model's own repository (it is named `sentencepiece.bpe.model` there) into the same directory. The loader tells you exactly this if the file is missing. diff --git a/opennlp-extensions/opennlp-embeddings/TRAINING.md b/opennlp-extensions/opennlp-embeddings/TRAINING.md index 08f8b3eb33..173e88fda5 100644 --- a/opennlp-extensions/opennlp-embeddings/TRAINING.md +++ b/opennlp-extensions/opennlp-embeddings/TRAINING.md @@ -17,46 +17,28 @@ # Distilling a Model for OpenNLP Static Embeddings -This module loads static embedding tables; it does not produce them. A table is distilled once from a sentence-transformer teacher, offline, in Python, and then loaded in the JVM as many times as you like. This walks through distilling one and assembling the directory `StaticEmbeddingModel.load` expects, using a multilingual SentencePiece model (bge-m3) as the worked example. +This module loads static embedding tables and, with the `DistillModel` command, also produces them: a table is distilled once from a sentence-transformer teacher and then loaded in the JVM as many times as you like. The distiller replicates [Model2Vec](https://github.com/MinishLab/model2vec) in Java — it runs the teacher's ONNX graph over its own vocabulary once, applies principal component analysis (PCA) and a Zipf weighting (frequent tokens are down-weighted, after Zipf's law of word frequency), and writes a flat per-token matrix. There is no training loop and no labelled data; a distillation is minutes on CPU, not hours on a GPU. -The distillation tool is [Model2Vec](https://github.com/MinishLab/model2vec). It runs the teacher over its own vocabulary once, applies principal component analysis (PCA) and a Zipf weighting (frequent tokens are down-weighted, after Zipf's law of word frequency), and writes a flat per-token matrix. There is no training loop and no labelled data; a distillation is minutes on CPU, not hours on a GPU. +## 1. Distill the teacher -## 1. Set up the distiller - -```bash -uv venv .venv-distill -uv pip install --python .venv-distill "model2vec[distill]" ``` - -## 2. Distill the teacher - -bge-m3 is an [XLM-RoBERTa](https://arxiv.org/abs/1911.02116)/SentencePiece model with a 250k multilingual vocabulary, native dimension 1024. - -```python -# distill_bge_m3.py -from model2vec.distill import distill - -static = distill("BAAI/bge-m3", pca_dims=256) -static.save_pretrained("bge-m3-static") -print("dim:", static.dim) +opennlp-embeddings DistillModel -teacher BAAI/bge-m3 -out bge-m3-static -pcaDims 256 ``` -```bash -.venv-distill/bin/python distill_bge_m3.py -``` +`-teacher` is a Hugging Face model id (its `tokenizer.json`, `tokenizer_config.json`, and `onnx/model.onnx` download once into `~/.cache/opennlp-embeddings`) or a local directory holding those files. `-pcaDims` defaults to 256. For a SentencePiece teacher like bge-m3 the trained `sentencepiece.bpe.model` is fetched alongside, because the static table keeps the teacher's segmentation. The command ends by completing the directory (the `AssembleModel` step) and verifying it by loading it, so a run that prints a summary is a directory that works. -This exact script lives in the repository as `dev/embeddings/distill_bge_m3.py`, and `dev/embeddings/parity/` holds a harness that reruns the parity check and the single-thread speed comparison against the Python reference on any machine. +bge-m3 is an [XLM-RoBERTa](https://arxiv.org/abs/1911.02116)/SentencePiece model with a 250k multilingual vocabulary, native dimension 1024. ### On the dimension -`pca_dims` is the one quality knob worth thinking about, and bigger is not better. Distilling bge-m3 at 256 and at 512 gives the same cross-lingual similarity within noise (English/Chinese paraphrase around 0.69 either way), while 512 doubles the matrix on disk and in memory and cuts embedding throughput. PCA to 256 already captures the useful variance of the teacher; the extra dimensions are mostly noise that dilutes the signal. 256 is a good default, and it is where the reference potion tables sit too. +`pcaDims` is the one quality knob worth thinking about, and bigger is not better. Distilling bge-m3 at 256 and at 512 gives the same cross-lingual similarity within noise (English/Chinese paraphrase around 0.69 either way), while 512 doubles the matrix on disk and in memory and cuts embedding throughput. PCA to 256 already captures the useful variance of the teacher; the extra dimensions are mostly noise that dilutes the signal. 256 is a good default, and it is where the reference potion tables sit too. -## 3. Assemble the model directory +## 2. Assemble the model directory -`save_pretrained` writes `model.safetensors`, `tokenizer.json`, and `config.json`, but not the trained SentencePiece `.model` file. That file is what actually segments text, so copy it from the teacher's own repository (on the Hub it is `sentencepiece.bpe.model`) into the same directory: +The distiller writes `model.safetensors` (F32), the cleaned `tokenizer.json`, and `config.json`, and copies the teacher's SentencePiece `.model` file when there is one. `DistillModel` assembles and verifies its own output; `AssembleModel` completes a directory put together by hand — for a WordPiece model it derives `vocab.txt` and `tokenizer_config.json` from `tokenizer.json`, for a SentencePiece model it checks that the trained `.model` file is present: -```bash -cp bge-m3-tokenizer/sentencepiece.bpe.model bge-m3-static/ +``` +opennlp-embeddings AssembleModel -modelDir bge-m3-static ``` A loadable SentencePiece directory then holds: @@ -65,23 +47,13 @@ A loadable SentencePiece directory then holds: bge-m3-static/ sentencepiece.bpe.model # copied from the teacher; segments the text tokenizer.json # Unigram vocab; its row order maps to the matrix - model.safetensors # the embedding matrix (F16 here, read natively) + model.safetensors # the embedding matrix config.json # carries "normalize": true|false ``` `load` detects the SentencePiece layout from the `.model` file next to `tokenizer.json`; it does not need `tokenizer_config.json`, because the `.model` carries the model's own text normalizer. If you forget the `.model` file, the loader says so by name. -### Let the tool assemble it - -Rather than assemble the directory by hand, run the `AssembleModel` command. It completes the directory in place and verifies it by loading it, so a run that prints a summary is a directory that works: - -``` -opennlp-embeddings AssembleModel -modelDir bge-m3-static -``` - -For a WordPiece distillation it derives the missing `vocab.txt` and `tokenizer_config.json` from `tokenizer.json` (the row order is the vocabulary in id order; the casing is the normalizer's lowercase flag). For a SentencePiece distillation it checks that the trained `.model` file is present and names the fix if it is not. Either way it prints the family, row count, and dimension of the loaded model. - -## 4. Load and verify in the JVM +## 3. Load and verify in the JVM ```java StaticEmbeddingModel model = StaticEmbeddingModel.load(Path.of("bge-m3-static")); @@ -96,13 +68,15 @@ double unrelated = model.similarity( model.mostSimilar("coffee", 5); // ▁coffee, ▁Coffee, ▁koffie, ▁kávé, ▁кофе ``` -Confirm parity against the Python reference before trusting a fresh distillation: embed the same text on both sides and check the vectors match within floating-point tolerance. They should agree to a few parts in ten thousand, because the JVM path reproduces the reference tokenization and pooling exactly, not approximately. +Confirm parity against the Python reference before trusting a fresh distillation: embed the same text on both sides and check the vectors match within floating-point tolerance. They should agree to a few parts in ten thousand, because the JVM path reproduces the reference tokenization and pooling exactly, not approximately. The reference Python flow lives in the repository as `dev/embeddings/distill_bge_m3.py`, and `dev/embeddings/parity/` holds a harness that reruns the parity check and the single-thread speed comparison against the Python reference on any machine. + +Two tables distilled independently from the same teacher (one with this command, one with Python Model2Vec) agree on their pairwise geometry to a few parts in a thousand — similarities, neighbors, and rankings match — but their raw vectors are not directly comparable axis by axis: PCA fixes only the subspace, and within the near-degenerate tail of the spectrum two independent decompositions choose different bases. ## The WordPiece path A WordPiece teacher (a BERT-family model such as bge-large-en) distills the same way. Its directory layout is the BERT one instead: `vocab.txt` (one token per line, line number is the row), `model.safetensors`, `config.json`, and `tokenizer_config.json` (whose `do_lower_case` sets the casing). `load` detects WordPiece from the presence of `vocab.txt`. -`save_pretrained` writes `tokenizer.json` rather than a `vocab.txt` for these, so derive `vocab.txt` from the `tokenizer.json` vocabulary in id order, and take `tokenizer_config.json` from the teacher for `do_lower_case`. +A distillation writes `tokenizer.json` rather than a `vocab.txt` for these, so run `AssembleModel` on the output directory: it derives `vocab.txt` from the `tokenizer.json` vocabulary in id order and `tokenizer_config.json` from the normalizer's lowercase flag. ## Where a table's license comes from diff --git a/opennlp-extensions/opennlp-embeddings/pom.xml b/opennlp-extensions/opennlp-embeddings/pom.xml index 7e827cf597..27f892340e 100644 --- a/opennlp-extensions/opennlp-embeddings/pom.xml +++ b/opennlp-extensions/opennlp-embeddings/pom.xml @@ -52,6 +52,13 @@ opennlp-cli + + + com.microsoft.onnxruntime + onnxruntime + ${onnxruntime.version} + + org.junit.jupiter junit-jupiter-api diff --git a/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java index b70234cee8..6593009a09 100644 --- a/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java +++ b/opennlp-extensions/opennlp-embeddings/src/jmh/java/opennlp/embeddings/StaticEmbeddingModelBenchmark.java @@ -46,14 +46,13 @@ import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; + import opennlp.embeddings.StaticEmbeddingModel.Casing; import opennlp.embeddings.StaticEmbeddingModel.Normalization; /** - * JMH benchmark for {@link StaticEmbeddingModel}, the raw-lookup-throughput number the module's - * design doc calls for before any "faster than Python" claim is made (a concurrent gRPC-traffic - * comparison against a Python baseline is a separate, later benchmark; this one is the JVM-only - * baseline). + * JMH benchmark for {@link StaticEmbeddingModel}: single-JVM embedding and neighbor-scan + * throughput. * *

The {@code modelDir} parameter selects the table to benchmark. Its default, * {@code "synthetic"}, builds a fixture sized to {@code minishlab/potion-base-8M} (29,528 rows, @@ -73,8 +72,7 @@ public class StaticEmbeddingModelBenchmark { /** The synthetic-fixture selector; any other value is treated as a model directory path. */ private static final String SYNTHETIC = "synthetic"; - // Matches minishlab/potion-base-8M's config.json (hidden_dim) and its reported total - // parameter count (7,559,168 / 256), verified against the real model repo, not guessed. + // Matches minishlab/potion-base-8M: hidden_dim 256 and 7,559,168 / 256 = 29,528 rows. private static final int VOCAB_SIZE = 29_528; private static final int DIMENSION = 256; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java index 86bd03a58c..98b2c698fc 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java @@ -59,12 +59,7 @@ private EmbeddingVocabulary(Map idByToken, List tokenBy * @throws IOException Thrown if reading the file fails. */ static EmbeddingVocabulary fromVocabTxt(Path file) throws IOException { - if (file == null) { - throw new IllegalArgumentException("File must not be null"); - } - if (!Files.isRegularFile(file)) { - throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); - } + requireRegularFile(file); return fromLines(Files.readAllLines(file), file.toString()); } @@ -79,13 +74,24 @@ static EmbeddingVocabulary fromVocabTxt(Path file) throws IOException { * @throws IOException Thrown if reading the file fails. */ static EmbeddingVocabulary fromTokenizerJson(Path file) throws IOException { + requireRegularFile(file); + return fromLines(TokenizerJsonVocab.rows(file), file.toString()); + } + + /** + * Requires {@code file} to be an existing regular file. + * + * @param file The file to check. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null}, missing, or not a + * regular file. + */ + private static void requireRegularFile(Path file) { if (file == null) { throw new IllegalArgumentException("File must not be null"); } if (!Files.isRegularFile(file)) { throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); } - return fromLines(TokenizerJsonVocab.rows(file), file.toString()); } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java index 4dd4dc83a5..11d1556722 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java @@ -23,11 +23,14 @@ /** * Reads single top-level fields out of a small flat JSON configuration file (a model's * {@code config.json} or {@code tokenizer_config.json}) without a JSON library dependency. Only - * top-level boolean look-ups are implemented; every other field is skipped structurally, and a + * top-level scalar look-ups are implemented; every other field is skipped structurally, and a * nested occurrence of the looked-up name never matches. */ final class FlatJsonFields { + /** The JSON null literal, accepted in place of any looked-up value. */ + private static final String NULL_LITERAL = "null"; + /** Not instantiable. */ private FlatJsonFields() { } @@ -45,6 +48,60 @@ private FlatJsonFields() { * @throws IOException Thrown if reading the file fails. */ static Boolean topLevelBoolean(Path file, String field) throws IOException { + return topLevelField(file, field, cursor -> { + if (cursor.consumeLiteral("true")) { + return Boolean.TRUE; + } + if (cursor.consumeLiteral("false")) { + return Boolean.FALSE; + } + if (cursor.consumeLiteral(NULL_LITERAL)) { + return null; + } + throw cursor.malformed("Field '" + field + "' must be a boolean or null"); + }); + } + + /** + * Reads one top-level string field from a JSON object file. + * + * @param file The JSON file, a single top-level object. Must not be {@code null} and must + * exist. + * @param field The top-level field name to read. Must not be {@code null}. + * @return The field's value, or {@code null} when the field is absent or explicitly JSON + * {@code null} (the formats treat those the same: fall back to the default). + * @throws IllegalArgumentException Thrown if the file is not a well-formed JSON object, the + * field appears more than once, or its value is neither a string nor {@code null}. + * @throws IOException Thrown if reading the file fails. + */ + static String topLevelString(Path file, String field) throws IOException { + return topLevelField(file, field, cursor -> { + if (cursor.consumeLiteral(NULL_LITERAL)) { + return null; + } + if (cursor.peek() == '"') { + return cursor.parseString(); + } + throw cursor.malformed("Field '" + field + "' must be a string or null"); + }); + } + + /** + * Walks a JSON object file's top-level fields, skipping every field but {@code field} and + * handing that one's value to {@code valueReader}. + * + * @param file The JSON file, a single top-level object. Must not be {@code null} and + * must exist. + * @param field The top-level field name to read. Must not be {@code null}. + * @param valueReader Reads the matched field's value off the cursor. + * @param The value type the reader produces. + * @return The field's value, or {@code null} when the field is absent. + * @throws IllegalArgumentException Thrown if an argument is {@code null}, the file is not a + * well-formed JSON object, or the field appears more than once. + * @throws IOException Thrown if reading the file fails. + */ + private static T topLevelField(Path file, String field, ValueReader valueReader) + throws IOException { if (file == null) { throw new IllegalArgumentException("File must not be null"); } @@ -56,7 +113,7 @@ static Boolean topLevelBoolean(Path file, String field) throws IOException { cursor.skipWhitespace(); cursor.expect('{'); cursor.skipWhitespace(); - Boolean value = null; + T value = null; boolean seen = false; if (cursor.peek() == '}') { cursor.consume(); @@ -72,13 +129,7 @@ static Boolean topLevelBoolean(Path file, String field) throws IOException { throw cursor.malformed("Field '" + field + "' appears more than once"); } seen = true; - if (cursor.consumeLiteral("true")) { - value = Boolean.TRUE; - } else if (cursor.consumeLiteral("false")) { - value = Boolean.FALSE; - } else if (!cursor.consumeLiteral("null")) { - throw cursor.malformed("Field '" + field + "' must be a boolean or null"); - } + value = valueReader.read(cursor); } else { cursor.skipValue(); } @@ -96,4 +147,22 @@ static Boolean topLevelBoolean(Path file, String field) throws IOException { cursor.requireEnd("Trailing content after the top-level object"); return value; } + + /** + * Decodes the value of the looked-up field, positioned at its first character. + * + * @param The value type produced. + */ + @FunctionalInterface + private interface ValueReader { + + /** + * Reads one value off the cursor. + * + * @param cursor The cursor, positioned at the value's first character. + * @return The decoded value, or {@code null} for a JSON {@code null}. + * @throws IllegalArgumentException Thrown if the value is not of the expected type. + */ + T read(JsonCursor cursor); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java new file mode 100644 index 0000000000..d9aaf36aa8 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Duration; + +/** + * Fetches the files a distillation needs from a Hugging Face model repository into a local cache + * directory, so a teacher can be named by its hub id ({@code org/model}) instead of a local path. + * Files download once and are reused afterwards; a file the repository does not have (a 404, e.g. + * a WordPiece teacher's {@code sentencepiece.bpe.model}) is reported as absent, not an error. + */ +final class HuggingFaceModelCache { + + /** The hub's file-download endpoint pattern: {@code BASE}/{id}/resolve/main/{file}. */ + private static final String RESOLVE_BASE = "https://huggingface.co/"; + + /** The ONNX graph of a hub transformer, relative to the repository root. */ + private static final String ONNX_MODEL = "onnx/model.onnx"; + + /** The files a distillation needs, relative to the repository root. */ + private static final String[] REQUIRED_FILES = {"tokenizer.json", ONNX_MODEL}; + + /** The files used when present: the pad-token config, the SentencePiece model, and the + * external weights of an ONNX export that splits them out (as bge-m3 does). */ + private static final String[] OPTIONAL_FILES = {"tokenizer_config.json", + "sentencepiece.bpe.model", "onnx/model.onnx_data"}; + + /** Not instantiable. */ + private HuggingFaceModelCache() { + } + + /** + * Resolves a teacher reference to a local directory holding its files. + * + * @param teacher A local directory, used as-is, or a Hugging Face model id + * ({@code org/model}), downloaded into + * {@code ~/.cache/opennlp-embeddings/} on first use. Must not be + * {@code null}. + * @param listener Receives one progress line per download; may be {@code null}. + * @return The local teacher directory. + * @throws IllegalArgumentException Thrown if {@code teacher} is {@code null}, a local path + * that is not a directory, or a hub id whose required files cannot be downloaded. + */ + static Path resolve(String teacher, ModelDistiller.ProgressListener listener) { + if (teacher == null) { + throw new IllegalArgumentException("Teacher must not be null"); + } + final Path local = Path.of(teacher); + if (Files.isDirectory(local)) { + return local; + } + if (!teacher.matches("[\\w.-]+/[\\w.-]+")) { + throw new IllegalArgumentException("Teacher '" + teacher + "' is neither a local " + + "directory nor a Hugging Face model id (expected 'org/model')"); + } + final Path cache = Path.of(System.getProperty("user.home"), ".cache", "opennlp-embeddings", + teacher.replace('/', '-').replace(".", "_")); + final HttpClient client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(Duration.ofSeconds(30)) + .build(); + for (final String file : REQUIRED_FILES) { + download(client, teacher, file, cache, true, listener); + } + for (final String file : OPTIONAL_FILES) { + download(client, teacher, file, cache, false, listener); + } + return cache; + } + + /** + * Downloads one repository file into the cache, skipping files already there. + * + * @param client The HTTP client. + * @param modelId The hub model id. + * @param file The repository-relative file name. + * @param cache The cache directory. + * @param required Whether a missing file is an error. + * @param listener The progress listener; may be {@code null}. + * @throws IllegalArgumentException Thrown if a required file cannot be downloaded. + */ + private static void download(HttpClient client, String modelId, String file, Path cache, + boolean required, ModelDistiller.ProgressListener listener) { + final Path target = cache.resolve(file); + if (Files.isRegularFile(target)) { + return; + } + final HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(RESOLVE_BASE + modelId + "/resolve/main/" + file)) + .timeout(Duration.ofHours(1)) + .GET() + .build(); + final HttpResponse response; + try { + response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to download " + file + " of " + modelId + ": " + + e.getMessage(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalArgumentException("Interrupted while downloading " + file + " of " + + modelId, e); + } + if (response.statusCode() != 200) { + if (required) { + throw new IllegalArgumentException("Failed to download " + file + " of " + modelId + + ": HTTP " + response.statusCode() + "; the distillation needs this file"); + } + return; + } + try { + if (listener != null) { + listener.progress("Downloading " + modelId + "/" + file + " ..."); + } + Files.createDirectories(target.getParent()); + final Path temporary = target.resolveSibling(target.getFileName() + ".download"); + try (InputStream body = response.body()) { + Files.copy(body, temporary, StandardCopyOption.REPLACE_EXISTING); + } + Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new IllegalArgumentException("Failed to store " + file + " of " + modelId + " at " + + target + ": " + e.getMessage(), e); + } + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java index 294595276d..91c3fba0f3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java @@ -18,8 +18,9 @@ /** * Cursor primitives shared by this package's purpose-built JSON readers - * ({@link SafetensorsHeaderParser}, {@link FlatJsonFields}): string and integer scalars, - * literals, whitespace, and skipping one value of any type. Deliberately not a general JSON + * ({@link SafetensorsHeaderParser}, {@link FlatJsonFields}, {@link TokenizerJsonVocab}, + * {@link ModelAssembler}): string and integer scalars, literals, whitespace, and skipping one + * value of any type. Deliberately not a general JSON * library: no floating-point decoding, no document model; each reader drives the cursor over * its own known-shape input and fails loud on anything else, with the input's name and the * offending offset in every message. @@ -49,6 +50,11 @@ void skipWhitespace() { } } + /** {@return the cursor's current offset into the text, for readers that capture raw spans} */ + int position() { + return position; + } + /** * {@return the character at the cursor without advancing} * @@ -85,7 +91,13 @@ void expect(char c) { } } - /** Consumes the given literal (e.g. {@code "true"}) if it starts here; returns whether. */ + /** + * Consumes the given literal (for example {@code "true"}) when it starts at the cursor, + * leaving the cursor untouched when it does not. + * + * @param literal The literal to match. + * @return {@code true} when the literal was consumed. + */ boolean consumeLiteral(String literal) { if (text.startsWith(literal, position)) { position += literal.length(); @@ -94,7 +106,12 @@ boolean consumeLiteral(String literal) { return false; } - /** Requires the rest of the input to be whitespace only. */ + /** + * Requires the rest of the input to be whitespace only. + * + * @param message What to report when other content follows. + * @throws IllegalArgumentException Thrown if non-whitespace content follows the cursor. + */ void requireEnd(String message) { skipWhitespace(); if (position < text.length()) { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java index b92ec01ef8..e8d4bba9c9 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java @@ -149,7 +149,8 @@ private static Result assembleWordpiece(Path modelDirectory, TokenizerJson token * @throws IOException Thrown if loading fails to read a file. */ private static Result assembleSentencePiece(Path modelDirectory) throws IOException { - if (firstExisting(modelDirectory, ModelFileNames.SENTENCEPIECE_MODELS) == null) { + if (ModelFileNames.firstRegularFile(modelDirectory, + ModelFileNames.SENTENCEPIECE_MODELS) == null) { throw new IllegalArgumentException("Model directory " + modelDirectory + " is a " + FAMILY_SENTENCEPIECE + " model but has no trained model file (one of " + String.join(", ", ModelFileNames.SENTENCEPIECE_MODELS) + "); copy it from the " @@ -194,22 +195,6 @@ private static Path requireFile(Path directory, String name) { return file; } - /** - * {@return the first of the given names that exists in the directory, or {@code null}} - * - * @param directory The directory to look in. - * @param names The names to try, in order. - */ - private static Path firstExisting(Path directory, List names) { - for (final String name : names) { - final Path file = directory.resolve(name); - if (Files.isRegularFile(file)) { - return file; - } - } - return null; - } - /** * The fields read out of a {@code tokenizer.json} for assembly. * diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java new file mode 100644 index 0000000000..c6fb15e9d5 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +/** + * Distills a sentence-transformer teacher into a static embedding table in the layout + * {@link StaticEmbeddingModel#load(Path)} opens, reproducing + * Model2Vec's distillation in Java so no + * Python environment is needed. The pipeline is Model2Vec's: + * + *

    + *
  1. The teacher's vocabulary is cleaned (unused tokens and special added tokens other than + * the unknown and pad tokens are dropped, the rest keeps its id order) and every surviving + * token is run through the teacher's ONNX graph as {@code [bos, token, eos]}; the token's + * embedding is the mean of the last hidden states.
    2. + *
  2. The matrix is projected onto its top principal components (a randomized SVD standing in + * for scikit-learn's dense one; see {@link RandomizedPca}).
    3. + *
  3. Each row is scaled by its Zipf weight {@code sif / (sif + p)}, where {@code p} is the + * row's share of a Zipf distribution over the vocabulary and {@code sif} is + * {@value #SIF_COEFFICIENT}, Model2Vec's default.
    4. + *
  4. The result is written as {@code model.safetensors} (F32), the cleaned + * {@code tokenizer.json}, and a {@code config.json} with {@code "normalize": true}; a + * SentencePiece teacher's {@code .model} file is copied alongside. The directory is then + * completed and verified by {@link ModelAssembler}.
    5. + *
+ * + *

The teacher directory must hold {@code tokenizer.json} and {@code onnx/model.onnx} (the + * ONNX export every sentence-transformer ships on the Hugging Face hub); a local + * {@code tokenizer_config.json} supplies the pad token when present.

+ */ +public final class ModelDistiller { + + /** Model2Vec's default SIF coefficient for the Zipf weighting. */ + static final double SIF_COEFFICIENT = 1e-4; + + /** The number of id sequences per forward-pass batch, Model2Vec's batch size. */ + private static final int BATCH_SIZE = 256; + + /** The fixed seed of the PCA range finder, so a distillation is reproducible. */ + private static final long PCA_SEED = 42; + + /** The ONNX graph inside a teacher directory. */ + private static final String ONNX_MODEL = "onnx/model.onnx"; + + /** Not instantiable. */ + private ModelDistiller() { + } + + /** Receives progress messages; the command-line tool prints them. */ + public interface ProgressListener { + + /** + * Reports a progress message. + * + * @param message The message. + */ + void progress(String message); + } + + /** + * The outcome of a distillation: the family, size, and dimension of the verified model, plus + * the variance the PCA kept. + * + * @param family {@code "WordPiece"} or {@code "SentencePiece"}. + * @param vocabularySize The number of rows in the distilled table. + * @param teacherDimension The teacher's hidden dimension. + * @param dimension The distilled table's dimension (after PCA). + * @param explainedVarianceRatio The share of the embedding variance the PCA kept. + */ + public record Result(String family, int vocabularySize, int teacherDimension, int dimension, + double explainedVarianceRatio) { + } + + /** + * Distills a teacher into a model directory, resolving the teacher reference first: a local + * directory is used as-is, a Hugging Face model id ({@code org/model}) is downloaded into a + * local cache on first use. + * + * @param teacher The teacher: a local directory or a Hugging Face model id. Must not + * be {@code null}. + * @param outputDirectory The model directory to write. Must not be {@code null}. + * @param pcaDims The number of principal components to keep. + * @param listener Receives progress lines; may be {@code null}. + * @return The distillation result, read back from the verified directory. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or invalid, the + * teacher cannot be resolved, or the teacher cannot be run. + * @throws IOException Thrown if reading or writing a file fails. + */ + public static Result distill(String teacher, Path outputDirectory, int pcaDims, + ProgressListener listener) throws IOException { + return distill(HuggingFaceModelCache.resolve(teacher, listener), outputDirectory, pcaDims, + listener); + } + + /** + * Distills a teacher into a model directory. + * + * @param teacherDirectory The teacher's directory, holding {@code tokenizer.json} and + * {@code onnx/model.onnx}. Must not be {@code null} and must be a + * directory. + * @param outputDirectory The model directory to write. Created when missing; an existing + * directory's distillation files are replaced. Must not be + * {@code null}. + * @param pcaDims The number of principal components to keep; clamped to the teacher's + * hidden dimension, and skipped entirely when it would not reduce a + * tiny vocabulary. Model2Vec's default (and the recommended value) is + * 256. + * @param listener Receives one progress line per forward-pass batch; may be + * {@code null}. + * @return The distillation result, read back from the verified directory. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or invalid, the + * teacher directory lacks its files, or the teacher cannot be run. + * @throws IOException Thrown if reading or writing a file fails. + */ + public static Result distill(Path teacherDirectory, Path outputDirectory, int pcaDims, + ProgressListener listener) + throws IOException { + if (teacherDirectory == null) { + throw new IllegalArgumentException("TeacherDirectory must not be null"); + } + if (!Files.isDirectory(teacherDirectory)) { + throw new IllegalArgumentException("Teacher directory does not exist or is not a " + + "directory: " + teacherDirectory); + } + if (outputDirectory == null) { + throw new IllegalArgumentException("OutputDirectory must not be null"); + } + if (pcaDims < 1) { + throw new IllegalArgumentException("PcaDims must be at least 1, got " + pcaDims); + } + final Path onnxFile = teacherDirectory.resolve(ONNX_MODEL); + if (!Files.isRegularFile(onnxFile)) { + throw new IllegalArgumentException("Teacher directory " + teacherDirectory + " has no " + + ONNX_MODEL + "; the distillation runs the teacher's ONNX export, which " + + "sentence-transformers ship on the Hugging Face hub"); + } + final TeacherTokenizer tokenizer = TeacherTokenizer.read( + teacherDirectory.resolve(ModelFileNames.TOKENIZER_JSON), + teacherDirectory.resolve(ModelFileNames.TOKENIZER_CONFIG)); + final int rows = tokenizer.vocabularySize(); + + final float[] embeddings; + final int teacherDimension; + try (OnnxTeacherEncoder encoder = OnnxTeacherEncoder.load(onnxFile)) { + float[][] first = encoder.encodeBatch(new long[][] {tokenizer.inputSequence(0)}); + teacherDimension = first[0].length; + embeddings = new float[rows * teacherDimension]; + System.arraycopy(first[0], 0, embeddings, 0, teacherDimension); + int row = 1; + while (row < rows) { + final int batchSize = Math.min(BATCH_SIZE, rows - row); + final long[][] batch = new long[batchSize][]; + for (int b = 0; b < batchSize; b++) { + batch[b] = tokenizer.inputSequence(row + b); + } + final float[][] pooled = encoder.encodeBatch(batch); + for (int b = 0; b < batchSize; b++) { + System.arraycopy(pooled[b], 0, embeddings, (row + b) * teacherDimension, + teacherDimension); + } + row += batchSize; + if (listener != null) { + listener.progress("Encoded " + row + " / " + rows + " vocabulary tokens"); + } + } + } + nanToZero(embeddings); + + final int components = Math.min(pcaDims, teacherDimension); + final float[] transformed; + double explainedVarianceRatio = 1.0; + if (components >= rows) { + // A PCA with more components than rows is not a reduction; Model2Vec skips it with a + // warning. Only reachable for toy vocabularies. + transformed = embeddings; + } else { + final RandomizedPca.Result pca = RandomizedPca.fitTransform(embeddings, rows, + teacherDimension, components, PCA_SEED); + transformed = pca.transformed(); + explainedVarianceRatio = pca.explainedVarianceRatio(); + } + final float[] weights = zipfWeights(rows, SIF_COEFFICIENT); + for (int row = 0; row < rows; row++) { + final int base = row * components; + final float weight = weights[row]; + for (int d = 0; d < components; d++) { + transformed[base + d] *= weight; + } + } + + Files.createDirectories(outputDirectory); + SafetensorsWriter.writeMatrix(outputDirectory.resolve(ModelFileNames.SAFETENSORS), rows, + components, transformed); + tokenizer.writeCleaned(outputDirectory.resolve(ModelFileNames.TOKENIZER_JSON)); + Files.writeString(outputDirectory.resolve(ModelFileNames.CONFIG), + configJson(teacherDirectory, pcaDims, components)); + copySentencePieceModel(teacherDirectory, outputDirectory); + final ModelAssembler.Result assembled = ModelAssembler.assemble(outputDirectory); + return new Result(assembled.family(), assembled.vocabularySize(), teacherDimension, + assembled.dimension(), explainedVarianceRatio); + } + + /** + * {@return Model2Vec's Zipf weights: row {@code i} gets {@code sif / (sif + p_i)} with + * {@code p_i = (1 / (i + 2)) / sum_j (1 / (j + 2))}, a SIF weighting under the assumption that + * vocabulary order approximates frequency order (Zipf's law)} + * + * @param rows The number of rows. + * @param sifCoefficient The SIF coefficient. + */ + static float[] zipfWeights(int rows, double sifCoefficient) { + double harmonicSum = 0; + for (int j = 2; j <= rows + 1; j++) { + harmonicSum += 1.0 / j; + } + final float[] weights = new float[rows]; + for (int i = 0; i < rows; i++) { + final double probability = 1.0 / (i + 2) / harmonicSum; + weights[i] = (float) (sifCoefficient / (sifCoefficient + probability)); + } + return weights; + } + + /** + * Replaces NaN values with zero, Model2Vec's {@code nan_to_num} guard against a teacher + * emitting a non-finite hidden state. + * + * @param values The matrix, modified in place. + */ + private static void nanToZero(float[] values) { + for (int i = 0; i < values.length; i++) { + if (Float.isNaN(values[i])) { + values[i] = 0; + } + } + } + + /** + * {@return the {@code config.json} of the distilled model, mirroring the fields Model2Vec + * writes; the loader reads only {@code normalize}} + * + * @param teacherDirectory The teacher's directory, for the name. + * @param pcaDims The requested PCA dimension. + * @param components The effective PCA dimension. + */ + private static String configJson(Path teacherDirectory, int pcaDims, int components) { + final Path name = teacherDirectory.getFileName(); + return "{\n" + + " \"model_type\": \"model2vec\",\n" + + " \"architectures\": [\"StaticModel\"],\n" + + " \"tokenizer_name\": \"" + (name == null ? teacherDirectory : name) + "\",\n" + + " \"apply_pca\": " + pcaDims + ",\n" + + " \"sif_coefficient\": " + SIF_COEFFICIENT + ",\n" + + " \"hidden_dim\": " + components + ",\n" + + " \"seq_length\": 1000000,\n" + + " \"normalize\": true,\n" + + " \"pooling\": \"mean\",\n" + + " \"embedding_dtype\": \"float32\"\n" + + "}\n"; + } + + /** + * Copies the teacher's trained SentencePiece {@code .model} file into the model directory when + * the teacher has one; the distillation cannot fabricate it and the loader needs it for the + * SentencePiece layout. + * + * @param teacherDirectory The teacher's directory. + * @param outputDirectory The model directory. + * @throws IOException Thrown if copying fails. + */ + private static void copySentencePieceModel(Path teacherDirectory, Path outputDirectory) + throws IOException { + for (final String name : ModelFileNames.SENTENCEPIECE_MODELS) { + final Path source = teacherDirectory.resolve(name); + if (Files.isRegularFile(source)) { + Files.copy(source, outputDirectory.resolve(name), + StandardCopyOption.REPLACE_EXISTING); + return; + } + } + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java index d0d2e1d543..cf7f6a4e0a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java @@ -16,6 +16,8 @@ */ package opennlp.embeddings; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; /** @@ -46,6 +48,23 @@ final class ModelFileNames { static final List SENTENCEPIECE_MODELS = List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); + /** + * {@return the first of the given file names that exists as a regular file in the directory, + * or {@code null} when none does} + * + * @param directory The directory to look in. + * @param names The file names to try, in order. + */ + static Path firstRegularFile(Path directory, List names) { + for (final String name : names) { + final Path file = directory.resolve(name); + if (Files.isRegularFile(file)) { + return file; + } + } + return null; + } + /** Not instantiable. */ private ModelFileNames() { } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java index b37a9b7808..d136f1703d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java @@ -16,12 +16,12 @@ */ package opennlp.embeddings; - /** * One vocabulary token found near a query vector by {@link StaticEmbeddingModel#mostSimilar} * or {@link StaticEmbeddingModel#analogy}, most similar first. * - * @param token The vocabulary token (a single WordPiece, not necessarily a whole word). + * @param token The vocabulary token: one subword piece of the model's tokenizer, which is + * not necessarily a whole word. * @param similarity Cosine similarity to the query vector, in {@code [-1, 1]}. */ public record Neighbor(String token, double similarity) { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java new file mode 100644 index 0000000000..d90583ca69 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import ai.onnxruntime.NodeInfo; +import ai.onnxruntime.OnnxTensor; +import ai.onnxruntime.OnnxValue; +import ai.onnxruntime.OrtEnvironment; +import ai.onnxruntime.OrtException; +import ai.onnxruntime.OrtSession; +import ai.onnxruntime.TensorInfo; + +/** + * Runs a teacher transformer over id sequences through its ONNX graph and mean-pools the last + * hidden states, the forward pass Model2Vec's distillation performs per vocabulary token. The + * graph is fed exactly the inputs it declares: {@code input_ids} and {@code attention_mask} for + * every model, plus a zero {@code token_type_ids} for the BERT-family graphs that ask for one. + * The pooled output is the mask-weighted mean of the single rank-3 float output (the + * {@code last_hidden_state}), over the non-padding positions only. + * + *

Not thread-safe; a distillation drives one instance from a single thread. Close it to + * release the native session.

+ */ +final class OnnxTeacherEncoder implements AutoCloseable { + + private final OrtEnvironment environment; + private final OrtSession session; + private final boolean wantsTokenTypeIds; + private final String hiddenStateOutput; + + /** Holds the open session; created by {@link #load(Path)}. */ + private OnnxTeacherEncoder(OrtEnvironment environment, OrtSession session, + boolean wantsTokenTypeIds, String hiddenStateOutput) { + this.environment = environment; + this.session = session; + this.wantsTokenTypeIds = wantsTokenTypeIds; + this.hiddenStateOutput = hiddenStateOutput; + } + + /** + * Loads a teacher's ONNX graph. + * + * @param onnxFile The ONNX file. Must not be {@code null} and must exist, must declare an + * {@code input_ids} input, and must produce exactly the rank-3 float + * last-hidden-state output this encoder pools. + * @return The encoder. + * @throws IllegalArgumentException Thrown if the file is missing, the graph has no + * {@code input_ids} input or no rank-3 float output, or the runtime rejects the graph. + */ + static OnnxTeacherEncoder load(Path onnxFile) { + if (onnxFile == null) { + throw new IllegalArgumentException("OnnxFile must not be null"); + } + if (!Files.isRegularFile(onnxFile)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + + onnxFile); + } + try { + final OrtEnvironment environment = OrtEnvironment.getEnvironment(); + final OrtSession session = environment.createSession(onnxFile.toString(), + new OrtSession.SessionOptions()); + if (!session.getInputNames().contains("input_ids")) { + session.close(); + throw new IllegalArgumentException("ONNX graph " + onnxFile + " has no 'input_ids' " + + "input; it does not look like a transformer encoder (inputs: " + + session.getInputNames() + ")"); + } + final boolean wantsTokenTypeIds = session.getInputNames().contains("token_type_ids"); + String hiddenStateOutput = null; + for (final Map.Entry output : session.getOutputInfo().entrySet()) { + if (output.getValue().getInfo() instanceof TensorInfo tensorInfo + && tensorInfo.getShape().length == 3) { + hiddenStateOutput = output.getKey(); + break; + } + } + if (hiddenStateOutput == null) { + session.close(); + throw new IllegalArgumentException("ONNX graph " + onnxFile + " has no rank-3 tensor " + + "output (a last hidden state) to pool (outputs: " + + session.getOutputInfo().keySet() + ")"); + } + return new OnnxTeacherEncoder(environment, session, wantsTokenTypeIds, hiddenStateOutput); + } catch (OrtException e) { + throw new IllegalArgumentException("Failed to load ONNX graph " + onnxFile + ": " + + e.getMessage(), e); + } + } + + /** + * Runs one batch of id sequences and mean-pools each sequence's hidden states. All sequences + * in a batch must have the same length (the distillation wraps every vocabulary token in the + * same bos/eos pair, so they do); the attention mask is all ones and no padding is needed. + * + * @param batch The id sequences, {@code [batchSize][sequenceLength]}. Must not be + * {@code null} or empty. + * @return The pooled vectors, {@code [batchSize][hiddenDimension]}. + * @throws IllegalArgumentException Thrown if the batch is empty or ragged, or the runtime + * rejects the input. + */ + float[][] encodeBatch(long[][] batch) { + if (batch == null || batch.length == 0) { + throw new IllegalArgumentException("Batch must not be null or empty"); + } + final int sequenceLength = batch[0].length; + for (final long[] sequence : batch) { + if (sequence.length != sequenceLength) { + throw new IllegalArgumentException("Batch is ragged: sequence lengths differ"); + } + } + final long[][] attentionMask = new long[batch.length][sequenceLength]; + for (final long[] mask : attentionMask) { + Arrays.fill(mask, 1L); + } + try { + final Map inputs = new HashMap<>(); + OnnxTensor tokenTypeIds = null; + try (OnnxTensor inputIds = OnnxTensor.createTensor(environment, batch); + OnnxTensor mask = OnnxTensor.createTensor(environment, attentionMask)) { + inputs.put("input_ids", inputIds); + inputs.put("attention_mask", mask); + if (wantsTokenTypeIds) { + tokenTypeIds = OnnxTensor.createTensor(environment, + new long[batch.length][sequenceLength]); + inputs.put("token_type_ids", tokenTypeIds); + } + try (OrtSession.Result result = session.run(inputs)) { + final OnnxValue value = result.get(hiddenStateOutput) + .orElseThrow(() -> new IllegalStateException("Output '" + hiddenStateOutput + + "' missing from the graph's results")); + final float[][][] hidden = (float[][][]) value.getValue(); + final float[][] pooled = new float[batch.length][]; + for (int i = 0; i < batch.length; i++) { + final float[][] states = hidden[i]; + final float[] sum = new float[states[0].length]; + for (final float[] state : states) { + for (int d = 0; d < sum.length; d++) { + sum[d] += state[d]; + } + } + for (int d = 0; d < sum.length; d++) { + sum[d] /= states.length; + if (Float.isNaN(sum[d])) { + sum[d] = 0; + } + } + pooled[i] = sum; + } + return pooled; + } finally { + if (tokenTypeIds != null) { + tokenTypeIds.close(); + } + } + } + } catch (OrtException e) { + throw new IllegalArgumentException("ONNX forward pass failed: " + e.getMessage(), e); + } + } + + /** Closes the native session. */ + @Override + public void close() { + try { + session.close(); + environment.close(); + } catch (OrtException e) { + // Closing a native resource must not mask a distillation result. + } + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java new file mode 100644 index 0000000000..42352422e3 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java @@ -0,0 +1,601 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.Random; +import java.util.function.IntConsumer; +import java.util.stream.IntStream; + +/** + * Principal component analysis by randomized SVD (Halko, Martinsson, Tropp), the approximation + * Model2Vec's distillation performs with a dense LAPACK SVD through scikit-learn. A dense SVD of + * a vocabulary-size matrix (250k rows for a multilingual teacher) is not practical in pure Java, + * so the top components are found with a random range finder and two power iterations, which for + * the fast-decaying spectrum of transformer token embeddings recovers the same subspace as the + * exact decomposition. + * + *

The column mean is subtracted before decomposition (the data matrix is modified in place), + * and the signs of the components are fixed the way scikit-learn's full solver fixes them + * ({@code svd_flip} with {@code u_based_decision=false}): each component's largest-magnitude + * coordinate is positive, so two distillations of the same teacher produce directly comparable + * vectors instead of mirror images.

+ * + *

The heavy loops are row-parallel over the common fork/join pool; all accumulation is in + * {@code double}.

+ */ +final class RandomizedPca { + + /** Extra dimensions the range finder samples beyond the requested components. */ + private static final int OVERSAMPLING = 10; + + /** Power iterations sharpening the range finder toward the dominant subspace. */ + private static final int POWER_ITERATIONS = 8; + + /** Number of row blocks the parallel loops split the matrix into. */ + private static final int BLOCKS = 32; + + /** Jacobi eigensolver convergence, relative to the largest diagonal element. */ + private static final double JACOBI_EPSILON = 1e-12; + + /** Jacobi eigensolver sweep cap; convergence arrives long before this. */ + private static final int JACOBI_MAX_SWEEPS = 100; + + /** Not instantiable. */ + private RandomizedPca() { + } + + /** The outcome of a PCA: the projected data and how much variance the projection keeps. */ + record Result(float[] transformed, double explainedVarianceRatio) { + } + + /** + * Centers {@code data} and projects it onto its top {@code components} principal components. + * + * @param data The row-major {@code rows x cols} matrix; centered in place. + * @param rows The number of rows. + * @param cols The number of columns (the original dimension). + * @param components The number of principal components to keep; at most {@code cols} and less + * than {@code rows}. + * @param seed The random seed of the range finder; a fixed seed makes the projection + * deterministic. + * @return The projected row-major {@code rows x components} matrix and the ratio of total + * variance it explains. + * @throws IllegalArgumentException Thrown if the arguments are inconsistent. + */ + static Result fitTransform(float[] data, int rows, int cols, int components, long seed) { + if (data == null) { + throw new IllegalArgumentException("Data must not be null"); + } + if (rows < 1 || cols < 1 || data.length != (long) rows * cols) { + throw new IllegalArgumentException("Data has " + data.length + " elements, not " + rows + + " x " + cols); + } + if (components < 1 || components > cols || components >= rows) { + throw new IllegalArgumentException("Components must be in [1, " + Math.min(cols, rows - 1) + + "], got " + components); + } + final double[] mean = columnMean(data, rows, cols); + subtractMean(data, rows, cols, mean); + final int sampleDimensions = Math.min(components + OVERSAMPLING, cols); + final double[] omega = new double[cols * sampleDimensions]; + final Random random = new Random(seed); + for (int i = 0; i < omega.length; i++) { + omega[i] = random.nextGaussian(); + } + double[] sample = multiplyDataByDense(data, rows, cols, omega, sampleDimensions); + for (int iteration = 0; iteration < POWER_ITERATIONS; iteration++) { + orthonormalizeInPlace(sample, rows, sampleDimensions); + final double[] transposed = multiplyDataTransposedByDense(data, rows, cols, sample, + sampleDimensions); + sample = multiplyDataByDense(data, rows, cols, transposed, sampleDimensions); + } + orthonormalizeInPlace(sample, rows, sampleDimensions); + // The small matrix B = Q'X holds the data's action on the found subspace; its right singular + // vectors rotated back are the principal components. + final double[] small = multiplyBasisTransposedByData(sample, rows, sampleDimensions, + data, cols); + final double[] gram = new double[sampleDimensions * sampleDimensions]; + for (int a = 0; a < sampleDimensions; a++) { + for (int b = 0; b <= a; b++) { + double sum = 0; + for (int c = 0; c < cols; c++) { + sum += small[a * cols + c] * small[b * cols + c]; + } + gram[a * sampleDimensions + b] = sum; + gram[b * sampleDimensions + a] = sum; + } + } + final double[][] eigen = jacobiEigen(gram, sampleDimensions); + final double[] eigenvalues = eigen[0]; + final double[] eigenvectors = eigen[1]; // row-major, column j is eigenvector j + // Components in component-major layout: component j is eigenvector j of B's Gram matrix + // mapped back through B and normalized by its singular value. + final double[] componentsMajor = new double[components * cols]; + for (int j = 0; j < components; j++) { + final double singularValue = Math.sqrt(Math.max(eigenvalues[j], JACOBI_EPSILON)); + for (int c = 0; c < cols; c++) { + double sum = 0; + for (int a = 0; a < sampleDimensions; a++) { + sum += small[a * cols + c] * eigenvectors[a * sampleDimensions + j]; + } + componentsMajor[j * cols + c] = sum / singularValue; + } + fixSign(componentsMajor, j * cols, cols); + } + final float[] transformed = project(data, rows, cols, componentsMajor, components); + double keptVariance = 0; + for (int j = 0; j < components; j++) { + keptVariance += eigenvalues[j]; + } + return new Result(transformed, keptVariance / totalVariance(data, rows, cols)); + } + + /** + * Fixes a component's sign the way scikit-learn's {@code svd_flip} with + * {@code u_based_decision=false} does: the largest-magnitude coordinate is made positive. + * + * @param componentMajor The component-major components array. + * @param offset The component's start offset. + * @param length The component's length. + */ + private static void fixSign(double[] componentMajor, int offset, int length) { + int maxIndex = 0; + double maxAbs = 0; + for (int c = 0; c < length; c++) { + final double abs = Math.abs(componentMajor[offset + c]); + if (abs > maxAbs) { + maxAbs = abs; + maxIndex = c; + } + } + if (componentMajor[offset + maxIndex] < 0) { + for (int c = 0; c < length; c++) { + componentMajor[offset + c] = -componentMajor[offset + c]; + } + } + } + + /** + * {@return the per-column means of the matrix, computed row-parallel} + * + * @param data The row-major matrix. + * @param rows The number of rows. + * @param cols The number of columns. + */ + private static double[] columnMean(float[] data, int rows, int cols) { + final double[][] partials = new double[BLOCKS][cols]; + forBlocks(rows, block -> { + final double[] partial = partials[block]; + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + for (int c = 0; c < cols; c++) { + partial[c] += data[i * cols + c]; + } + } + }); + final double[] mean = new double[cols]; + for (final double[] partial : partials) { + for (int c = 0; c < cols; c++) { + mean[c] += partial[c]; + } + } + for (int c = 0; c < cols; c++) { + mean[c] /= rows; + } + return mean; + } + + /** + * Subtracts the per-column means from the matrix in place, row-parallel. + * + * @param data The row-major matrix. + * @param rows The number of rows. + * @param cols The number of columns. + * @param mean The per-column means. + */ + private static void subtractMean(float[] data, int rows, int cols, double[] mean) { + final float[] meanFloat = new float[cols]; + for (int c = 0; c < cols; c++) { + meanFloat[c] = (float) mean[c]; + } + forBlocks(rows, block -> { + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + for (int c = 0; c < cols; c++) { + data[i * cols + c] -= meanFloat[c]; + } + } + }); + } + + /** + * {@return the total variance of the centered matrix (the squared Frobenius norm), row-parallel} + * + * @param data The centered row-major matrix. + * @param rows The number of rows. + * @param cols The number of columns. + */ + private static double totalVariance(float[] data, int rows, int cols) { + final double[] partials = new double[BLOCKS]; + forBlocks(rows, block -> { + double sum = 0; + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start * cols; i < end * cols; i++) { + sum += (double) data[i] * data[i]; + } + partials[block] = sum; + }); + double total = 0; + for (final double partial : partials) { + total += partial; + } + return total; + } + + /** + * {@return the product {@code data * dense} of the float data matrix with a dense double + * matrix, row-parallel over the data} + * + * @param data The row-major {@code rows x cols} float matrix. + * @param rows The number of rows. + * @param cols The number of columns. + * @param dense The row-major {@code cols x width} double matrix. + * @param width The number of columns of {@code dense}. + */ + private static double[] multiplyDataByDense(float[] data, int rows, int cols, double[] dense, + int width) { + final double[] out = new double[rows * width]; + forBlocks(rows, block -> { + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + final int rowBase = i * cols; + final int outBase = i * width; + for (int c = 0; c < cols; c++) { + final float value = data[rowBase + c]; + if (value != 0) { + final int denseBase = c * width; + for (int j = 0; j < width; j++) { + out[outBase + j] += value * dense[denseBase + j]; + } + } + } + } + }); + return out; + } + + /** + * {@return the product {@code data' * dense} of the transposed float data matrix with a dense + * double matrix, row-parallel over the data with per-block partial results} + * + * @param data The row-major {@code rows x cols} float matrix. + * @param rows The number of rows. + * @param cols The number of columns. + * @param dense The row-major {@code rows x width} double matrix. + * @param width The number of columns of {@code dense}. + */ + private static double[] multiplyDataTransposedByDense(float[] data, int rows, int cols, + double[] dense, int width) { + final double[][] partials = new double[BLOCKS][cols * width]; + forBlocks(rows, block -> { + final double[] partial = partials[block]; + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + final int rowBase = i * cols; + final int denseBase = i * width; + for (int c = 0; c < cols; c++) { + final float value = data[rowBase + c]; + if (value != 0) { + final int outBase = c * width; + for (int j = 0; j < width; j++) { + partial[outBase + j] += value * dense[denseBase + j]; + } + } + } + } + }); + final double[] out = new double[cols * width]; + for (final double[] partial : partials) { + for (int i = 0; i < out.length; i++) { + out[i] += partial[i]; + } + } + return out; + } + + /** + * {@return the product {@code basis' * data} of the transposed orthonormal basis with the float + * data matrix, row-parallel over the data with per-block partial results} + * + * @param basis The row-major {@code rows x width} orthonormal basis. + * @param rows The number of rows. + * @param width The basis width. + * @param data The row-major {@code rows x cols} float matrix. + * @param cols The number of data columns. + */ + private static double[] multiplyBasisTransposedByData(double[] basis, int rows, int width, + float[] data, int cols) { + final double[][] partials = new double[BLOCKS][width * cols]; + forBlocks(rows, block -> { + final double[] partial = partials[block]; + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + final int basisBase = i * width; + final int rowBase = i * cols; + for (int a = 0; a < width; a++) { + final double value = basis[basisBase + a]; + final int outBase = a * cols; + for (int c = 0; c < cols; c++) { + partial[outBase + c] += value * data[rowBase + c]; + } + } + } + }); + final double[] out = new double[width * cols]; + for (final double[] partial : partials) { + for (int i = 0; i < out.length; i++) { + out[i] += partial[i]; + } + } + return out; + } + + /** + * {@return the projection {@code data * components'} onto the component-major components, + * row-parallel} + * + * @param data The centered row-major {@code rows x cols} float matrix. + * @param rows The number of rows. + * @param cols The number of columns. + * @param componentsMajor The row-major {@code components x cols} components. + * @param components The number of components. + */ + private static float[] project(float[] data, int rows, int cols, double[] componentsMajor, + int components) { + final float[] out = new float[rows * components]; + forBlocks(rows, block -> { + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + final int rowBase = i * cols; + final int outBase = i * components; + for (int j = 0; j < components; j++) { + final int componentBase = j * cols; + double sum = 0; + for (int c = 0; c < cols; c++) { + sum += data[rowBase + c] * componentsMajor[componentBase + c]; + } + out[outBase + j] = (float) sum; + } + } + }); + return out; + } + + /** + * Orthonormalizes the columns of the tall {@code rows x width} matrix in place by CholeskyQR: + * the Cholesky factor of the Gram matrix triangular-solves the basis. A diagonal jitter relative + * to the average pivot keeps the factorization alive when a power iteration has driven the + * columns toward linear dependence. + * + * @param matrix The row-major tall matrix, orthonormalized in place. + * @param rows The number of rows. + * @param width The number of columns. + */ + private static void orthonormalizeInPlace(double[] matrix, int rows, int width) { + final double[][] partials = new double[BLOCKS][width * width]; + forBlocks(rows, block -> { + final double[] partial = partials[block]; + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + final int base = i * width; + for (int a = 0; a < width; a++) { + final double value = matrix[base + a]; + final int gramBase = a * width; + for (int b = 0; b <= a; b++) { + partial[gramBase + b] += value * matrix[base + b]; + } + } + } + }); + final double[] gram = new double[width * width]; + for (final double[] partial : partials) { + for (int a = 0; a < width; a++) { + for (int b = 0; b <= a; b++) { + gram[a * width + b] += partial[a * width + b]; + } + } + } + for (int a = 0; a < width; a++) { + for (int b = 0; b < a; b++) { + gram[b * width + a] = gram[a * width + b]; + } + } + double trace = 0; + for (int a = 0; a < width; a++) { + trace += gram[a * width + a]; + } + double jitter = Math.max(trace / width, 1) * 1e-12; + double[] lower = null; + for (int attempt = 0; attempt < 5 && lower == null; attempt++) { + lower = cholesky(gram, width, jitter); + jitter *= 1000; + } + if (lower == null) { + throw new IllegalStateException("Gram matrix is not positive definite even with jitter; " + + "the data columns are linearly dependent"); + } + // Solve Q L' = Y row-wise. Transposed, that is L q' = y': a forward substitution against + // the lower-triangular L. + final double[] factor = lower; + forBlocks(rows, block -> { + final int start = blockStart(rows, block); + final int end = blockStart(rows, block + 1); + for (int i = start; i < end; i++) { + final int base = i * width; + for (int j = 0; j < width; j++) { + double sum = matrix[base + j]; + for (int m = 0; m < j; m++) { + sum -= factor[j * width + m] * matrix[base + m]; + } + matrix[base + j] = sum / factor[j * width + j]; + } + } + }); + } + + /** + * {@return the lower-triangular Cholesky factor of the symmetric positive-definite matrix, or + * {@code null} when a pivot is not positive even after adding {@code jitter} to the diagonal} + * + * @param matrix The row-major symmetric matrix. + * @param width The matrix order. + * @param jitter The value added to the diagonal before factoring. + */ + private static double[] cholesky(double[] matrix, int width, double jitter) { + final double[] lower = new double[width * width]; + for (int a = 0; a < width; a++) { + for (int b = 0; b <= a; b++) { + double sum = matrix[a * width + b]; + if (a == b) { + sum += jitter; + } + for (int m = 0; m < b; m++) { + sum -= lower[a * width + m] * lower[b * width + m]; + } + if (a == b) { + if (sum <= 0) { + return null; + } + lower[a * width + a] = Math.sqrt(sum); + } else { + lower[a * width + b] = sum / lower[b * width + b]; + } + } + } + return lower; + } + + /** + * {@return the eigenpairs of a small symmetric matrix by the cyclic Jacobi method, eigenvalues + * descending; the returned array holds the eigenvalues at index 0 and the row-major eigenvector + * matrix (column j is eigenvector j) at index 1} + * + * @param matrix The row-major symmetric matrix; not modified. + * @param width The matrix order. + */ + private static double[][] jacobiEigen(double[] matrix, int width) { + final double[] a = matrix.clone(); + final double[] eigenvectors = new double[width * width]; + for (int i = 0; i < width; i++) { + eigenvectors[i * width + i] = 1; + } + for (int sweep = 0; sweep < JACOBI_MAX_SWEEPS; sweep++) { + double offDiagonal = 0; + double diagonal = 0; + for (int p = 0; p < width; p++) { + diagonal = Math.max(diagonal, Math.abs(a[p * width + p])); + for (int q = p + 1; q < width; q++) { + offDiagonal += a[p * width + q] * a[p * width + q]; + } + } + if (Math.sqrt(offDiagonal) <= JACOBI_EPSILON * Math.max(diagonal, JACOBI_EPSILON)) { + break; + } + for (int p = 0; p < width; p++) { + for (int q = p + 1; q < width; q++) { + final double apq = a[p * width + q]; + if (Math.abs(apq) <= JACOBI_EPSILON * Math.max(diagonal, JACOBI_EPSILON)) { + continue; + } + final double app = a[p * width + p]; + final double aqq = a[q * width + q]; + final double theta = (aqq - app) / (2 * apq); + final double sign = theta >= 0 ? 1 : -1; + final double t = sign / (Math.abs(theta) + Math.sqrt(theta * theta + 1)); + final double cosine = 1 / Math.sqrt(t * t + 1); + final double sine = t * cosine; + for (int k = 0; k < width; k++) { + final double akp = a[k * width + p]; + final double akq = a[k * width + q]; + a[k * width + p] = cosine * akp - sine * akq; + a[k * width + q] = sine * akp + cosine * akq; + } + for (int k = 0; k < width; k++) { + final double apk = a[p * width + k]; + final double aqk = a[q * width + k]; + a[p * width + k] = cosine * apk - sine * aqk; + a[q * width + k] = sine * apk + cosine * aqk; + } + for (int k = 0; k < width; k++) { + final double vkp = eigenvectors[k * width + p]; + final double vkq = eigenvectors[k * width + q]; + eigenvectors[k * width + p] = cosine * vkp - sine * vkq; + eigenvectors[k * width + q] = sine * vkp + cosine * vkq; + } + } + } + } + // Sort eigenpairs by eigenvalue, descending, with an insertion sort (the matrix is small). + final double[] eigenvalues = new double[width]; + for (int j = 0; j < width; j++) { + eigenvalues[j] = a[j * width + j]; + } + for (int j = 1; j < width; j++) { + int k = j; + while (k > 0 && eigenvalues[k - 1] < eigenvalues[k]) { + final double value = eigenvalues[k]; + eigenvalues[k] = eigenvalues[k - 1]; + eigenvalues[k - 1] = value; + for (int i = 0; i < width; i++) { + final double v = eigenvectors[i * width + k]; + eigenvectors[i * width + k] = eigenvectors[i * width + k - 1]; + eigenvectors[i * width + k - 1] = v; + } + k--; + } + } + return new double[][] {eigenvalues, eigenvectors}; + } + + /** + * Runs {@code action} for every row-block index in parallel over the common pool. + * + * @param rows The total number of rows. + * @param action Receives the block index, in {@code [0, BLOCKS)}. + */ + private static void forBlocks(int rows, IntConsumer action) { + IntStream.range(0, Math.min(BLOCKS, rows)).parallel().forEach(action); + } + + /** + * {@return the first row of a block} + * + * @param rows The total number of rows. + * @param block The block index; the effective block count yields the end sentinel. + */ + private static int blockStart(int rows, int block) { + return (int) ((long) rows * block / Math.min(BLOCKS, rows)); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java new file mode 100644 index 0000000000..4db79e6a8c --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +/** + * Writes a safetensors file holding a + * single 2-D {@code F32} tensor, the shape a distilled embedding table takes (vocabulary size by + * output dimension). This is the write side of the format {@link SafetensorsFile} reads; the data + * is streamed to the file in chunks so the writer's overhead beyond the caller's matrix is + * constant. + */ +final class SafetensorsWriter { + + /** The name of the embedding matrix tensor, the Model2Vec convention. */ + static final String EMBEDDINGS_TENSOR = "embeddings"; + + // Encoding chunk, a multiple of Float.BYTES. + private static final int WRITE_CHUNK_BYTES = 1 << 20; + + /** Not instantiable. */ + private SafetensorsWriter() { + } + + /** + * Writes a row-major float matrix as a one-tensor safetensors file. + * + * @param file The file to write, replaced when it exists. Must not be {@code null}. + * @param rows The number of matrix rows. + * @param cols The number of matrix columns. + * @param values The matrix values in row-major order, {@code rows * cols} of them. Must not be + * {@code null}. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or the value count + * does not match the shape. + * @throws IOException Thrown if writing fails. + */ + static void writeMatrix(Path file, int rows, int cols, float[] values) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (values == null) { + throw new IllegalArgumentException("Values must not be null"); + } + if (rows < 1 || cols < 1 || values.length != (long) rows * cols) { + throw new IllegalArgumentException("Values has " + values.length + " elements, not " + rows + + " x " + cols); + } + final long dataBytes = (long) values.length * Float.BYTES; + final String header = "{\"" + EMBEDDINGS_TENSOR + "\":{\"dtype\":\"F32\",\"shape\":[" + rows + + "," + cols + "],\"data_offsets\":[0," + dataBytes + "]}}"; + final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final Path parent = file.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + final ByteBuffer prefix = ByteBuffer.allocate(8 + headerBytes.length) + .order(ByteOrder.LITTLE_ENDIAN); + prefix.putLong(headerBytes.length); + prefix.put(headerBytes); + prefix.flip(); + writeFully(channel, prefix); + final ByteBuffer chunk = ByteBuffer.allocate(WRITE_CHUNK_BYTES) + .order(ByteOrder.LITTLE_ENDIAN); + int written = 0; + while (written < values.length) { + chunk.clear(); + final int count = Math.min(values.length - written, WRITE_CHUNK_BYTES / Float.BYTES); + chunk.asFloatBuffer().put(values, written, count); + chunk.limit(count * Float.BYTES); + writeFully(channel, chunk); + written += count; + } + } + } + + /** + * Writes the buffer's remaining bytes to the channel. + * + * @param channel The open channel. + * @param buffer The buffer to drain. + * @throws IOException Thrown if writing fails. + */ + private static void writeFully(FileChannel channel, ByteBuffer buffer) throws IOException { + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 973f105c25..bd01721242 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -156,7 +156,7 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException if (Files.isRegularFile(vocabularyFile)) { return loadWordpieceDirectory(modelDirectory, vocabularyFile); } - final Path sentencePieceModelFile = firstRegularFile(modelDirectory, + final Path sentencePieceModelFile = ModelFileNames.firstRegularFile(modelDirectory, ModelFileNames.SENTENCEPIECE_MODELS); final Path tokenizerJsonFile = modelDirectory.resolve(ModelFileNames.TOKENIZER_JSON); if (sentencePieceModelFile != null && Files.isRegularFile(tokenizerJsonFile)) { @@ -230,23 +230,6 @@ private static Normalization requiredNormalize(Path configFile) throws IOExcepti return normalize ? Normalization.L2 : Normalization.NONE; } - /** - * {@return the first of the given file names that exists as a regular file in the directory, - * or {@code null} when none does} - * - * @param directory The directory to look in. - * @param names The file names to try, in order. - */ - private static Path firstRegularFile(Path directory, List names) { - for (final String name : names) { - final Path file = directory.resolve(name); - if (Files.isRegularFile(file)) { - return file; - } - } - return null; - } - /** * {@return the named file in the directory, requiring it to exist as a regular file} * @@ -703,7 +686,7 @@ public List analogy(String a, String b, String c, int topK) { * @param topK The requested result count. * @throws IllegalArgumentException Thrown if {@code topK} is less than 1. */ - private static void requirePositive(int topK) { + private void requirePositive(int topK) { if (topK < 1) { throw new IllegalArgumentException("TopK must be at least 1, got " + topK); } @@ -802,7 +785,7 @@ private List nearestNeighbors(float[] query, int topK, int[] sortedExc * @param a The first vector. * @param b The second vector, of the same length as {@code a}. */ - private static double cosineSimilarity(float[] a, float[] b) { + private double cosineSimilarity(float[] a, float[] b) { double dot = 0; double normASquared = 0; double normBSquared = 0; @@ -820,7 +803,7 @@ private static double cosineSimilarity(float[] a, float[] b) { * * @param vector The vector to measure. */ - private static double norm(float[] vector) { + private double norm(float[] vector) { double sumOfSquares = 0; for (final float value : vector) { sumOfSquares += (double) value * value; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java new file mode 100644 index 0000000000..15f6cb53d6 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java @@ -0,0 +1,1025 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * The tokenizer side of a teacher model, distilled the way Model2Vec distills it. The class reads + * the teacher's {@code tokenizer.json} (and, when present, its {@code tokenizer_config.json} for + * the pad token), decides which vocabulary rows survive into the static table, and rewrites the + * {@code tokenizer.json} so it describes the distilled table. + * + *

The cleaning mirrors Model2Vec: tokens matching {@code \[unused\d+\]} are removed, the + * added-token overlay is pruned to the unknown and pad tokens (the only special tokens a distilled + * table keeps), the post-processor is dropped (a static table is pooled from content pieces, never + * wrapped in {@code [CLS]}/{@code [SEP]}), and the surviving tokens keep their original id order + * but are renumbered to a gapless id space. That new order is the matrix row order.

+ * + *

For the forward pass the class reports, per surviving token, its id in the teacher's + * id space plus the teacher's begin/end-of-sequence wrapper ids: Model2Vec feeds each vocabulary + * token to the teacher as {@code [bos, token, eos]} and mean-pools the hidden states.

+ * + *

The rewrite copies every field it does not change byte for byte (the normalizer, the + * pre-tokenizer, the Unigram scores), so the cleaned {@code tokenizer.json} stays a faithful + * fast-tokenizer description of the distilled table.

+ */ +final class TeacherTokenizer { + + /** Model2Vec's default token removal pattern; matched from the start, like Python re.match. */ + private static final Pattern UNUSED_TOKEN_PATTERN = Pattern.compile("\\[unused\\d+\\]"); + + /** The WordPiece {@code model.type} of a BERT-family teacher. */ + static final String WORDPIECE = "WordPiece"; + + /** The Unigram {@code model.type} of a SentencePiece-family teacher. */ + static final String UNIGRAM = "Unigram"; + + private final String json; + private final String inputName; + private final String modelType; + private final List tokensByOriginalId; + private final int[] keptOriginalIds; + private final int originalUnkId; + private final String unkToken; + private final String padToken; + private final int padTokenId; + private final int[] bosIds; + private final int[] eosIds; + + /** Holds the parsed state; built by {@link #read(Path, Path)}. */ + private TeacherTokenizer(String json, String inputName, String modelType, + List tokensByOriginalId, int[] keptOriginalIds, + int originalUnkId, String unkToken, String padToken, int padTokenId, + int[] bosIds, int[] eosIds) { + this.json = json; + this.inputName = inputName; + this.modelType = modelType; + this.tokensByOriginalId = tokensByOriginalId; + this.keptOriginalIds = keptOriginalIds; + this.originalUnkId = originalUnkId; + this.unkToken = unkToken; + this.padToken = padToken; + this.padTokenId = padTokenId; + this.bosIds = bosIds; + this.eosIds = eosIds; + } + + /** + * Reads a teacher's tokenizer configuration. + * + * @param tokenizerJsonFile The teacher's {@code tokenizer.json}. Must not be {@code null} + * and must exist. + * @param tokenizerConfigFile The teacher's {@code tokenizer_config.json}, consulted for the + * pad token only; may be {@code null} (no pad token then). + * @return The parsed teacher tokenizer. + * @throws IllegalArgumentException Thrown if the files are missing or malformed, the tokenizer + * model is neither WordPiece nor Unigram, the vocabulary ids are not a gapless range, the + * unknown token is missing, or the post-processor is of an unsupported type. + * @throws IOException Thrown if reading a file fails. + */ + static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) + throws IOException { + if (tokenizerJsonFile == null) { + throw new IllegalArgumentException("TokenizerJsonFile must not be null"); + } + if (!Files.isRegularFile(tokenizerJsonFile)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + + tokenizerJsonFile); + } + final String padToken = tokenizerConfigFile != null && Files.isRegularFile(tokenizerConfigFile) + ? FlatJsonFields.topLevelString(tokenizerConfigFile, "pad_token") : null; + final String json = Files.readString(tokenizerJsonFile); + final String inputName = tokenizerJsonFile.getFileName().toString(); + final JsonCursor cursor = new JsonCursor(json, inputName); + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + String modelType = null; + List tokensById = null; + String unkToken = null; + Long unkId = null; + Set addedContents = Set.of(); + PostProcessor postProcessor = new PostProcessor(List.of(), List.of(), null, null, Map.of()); + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "model" -> { + final ModelSection model = parseModel(cursor); + modelType = model.type(); + tokensById = model.tokensById(); + unkToken = model.unkToken(); + unkId = model.unkId(); + } + case "added_tokens" -> addedContents = parseAddedTokenContents(cursor); + case "post_processor" -> postProcessor = parsePostProcessor(cursor); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a field, got '" + next + "'"); + } + } + cursor.requireEnd("Trailing content after the top-level object"); + if (modelType == null || tokensById == null) { + throw new IllegalArgumentException(tokenizerJsonFile + " has no model with a vocabulary; " + + "it does not look like a teacher's tokenizer.json"); + } + if (!WORDPIECE.equals(modelType) && !UNIGRAM.equals(modelType)) { + throw new IllegalArgumentException(tokenizerJsonFile + " has a '" + modelType + + "' tokenizer model; only " + WORDPIECE + " and " + UNIGRAM + + " teachers are supported"); + } + final Map idByToken = new HashMap<>(tokensById.size() * 2); + for (int id = 0; id < tokensById.size(); id++) { + idByToken.putIfAbsent(tokensById.get(id), id); + } + if (unkToken == null) { + if (unkId == null || unkId < 0 || unkId >= tokensById.size()) { + throw new IllegalArgumentException(tokenizerJsonFile + " does not name an unknown token " + + "(no model.unk_token / model.unk_id); a distilled table needs one"); + } + unkToken = tokensById.get(unkId.intValue()); + } + final Integer originalUnkId = idByToken.get(unkToken); + if (originalUnkId == null) { + throw new IllegalArgumentException(tokenizerJsonFile + " names the unknown token '" + + unkToken + "' but it is not in the vocabulary"); + } + // The wrapper ids come from the cls/sep pairs of a BertProcessing/RobertaProcessing + // post-processor, or from resolving a TemplateProcessing's special token names through its + // special_tokens table, falling back to the vocabulary. + final int[] bosIds = postProcessor.clsId() != null + ? new int[] {postProcessor.clsId().intValue()} + : resolveNames(postProcessor.bosNames(), postProcessor.specialTokenIds(), idByToken, + tokenizerJsonFile); + final int[] eosIds = postProcessor.sepId() != null + ? new int[] {postProcessor.sepId().intValue()} + : resolveNames(postProcessor.eosNames(), postProcessor.specialTokenIds(), idByToken, + tokenizerJsonFile); + final Integer padId = padToken == null ? null : idByToken.get(padToken); + final int padTokenId = padId == null ? 0 : padId; + final Set keepSpecial = new HashSet<>(); + keepSpecial.add(unkToken); + if (padToken != null) { + keepSpecial.add(padToken); + } + final List kept = new ArrayList<>(tokensById.size()); + for (int id = 0; id < tokensById.size(); id++) { + final String token = tokensById.get(id); + if (UNUSED_TOKEN_PATTERN.matcher(token).lookingAt()) { + continue; + } + if (addedContents.contains(token) && !keepSpecial.contains(token)) { + continue; + } + kept.add(id); + } + return new TeacherTokenizer(json, inputName, modelType, tokensById, + kept.stream().mapToInt(Integer::intValue).toArray(), originalUnkId, unkToken, padToken, + padTokenId, bosIds, eosIds); + } + + /** + * {@return the ids the named special tokens resolve to, through the post-processor's + * special-token table first and the vocabulary second} + * + * @param names The special token names in order. + * @param specialTokenIds The post-processor's name-to-id table. + * @param idByToken The vocabulary, token to id. + * @param file The source file, for error messages. + * @throws IllegalArgumentException Thrown if a name resolves nowhere. + */ + private static int[] resolveNames(List names, Map specialTokenIds, + Map idByToken, Path file) { + final int[] ids = new int[names.size()]; + for (int i = 0; i < names.size(); i++) { + final Long specialId = specialTokenIds.get(names.get(i)); + final Integer vocabId = idByToken.get(names.get(i)); + if (specialId != null) { + ids[i] = specialId.intValue(); + } else if (vocabId != null) { + ids[i] = vocabId; + } else { + throw new IllegalArgumentException(file + " wraps sequences in the special token '" + + names.get(i) + "' but neither the post-processor nor the vocabulary defines it"); + } + } + return ids; + } + + /** {@return the tokenizer family, {@code "WordPiece"} or {@code "Unigram"}} */ + String modelType() { + return modelType; + } + + /** {@return the number of surviving tokens, the matrix row count} */ + int vocabularySize() { + return keptOriginalIds.length; + } + + /** {@return the surviving tokens' ids in the teacher's id space, in matrix row order} */ + int[] keptOriginalIds() { + return keptOriginalIds.clone(); + } + + /** {@return the teacher's pad token id, used to pad batches; 0 when the teacher names none} */ + int padTokenId() { + return padTokenId; + } + + /** {@return the unknown token's string} */ + String unkToken() { + return unkToken; + } + + /** {@return the pad token's string, or {@code null} when the teacher names none} */ + String padToken() { + return padToken; + } + + /** + * The teacher input sequence for one matrix row: the begin-of-sequence ids, the token's + * original id, and the end-of-sequence ids. + * + * @param row The matrix row. + * @return The teacher input ids. + */ + long[] inputSequence(int row) { + final long[] sequence = new long[bosIds.length + 1 + eosIds.length]; + int i = 0; + for (final int id : bosIds) { + sequence[i++] = id; + } + sequence[i++] = keptOriginalIds[row]; + for (final int id : eosIds) { + sequence[i++] = id; + } + return sequence; + } + + /** + * Writes the cleaned {@code tokenizer.json}: the surviving vocabulary renumbered, the + * added-token overlay pruned to the unknown and pad tokens, the post-processor nulled, and + * every other field copied byte for byte from the teacher's file. + * + * @param file The file to write. Must not be {@code null}. + * @throws IOException Thrown if writing fails. + */ + void writeCleaned(Path file) throws IOException { + final Map newIdByOriginal = new HashMap<>(keptOriginalIds.length * 2); + for (int row = 0; row < keptOriginalIds.length; row++) { + newIdByOriginal.put(keptOriginalIds[row], row); + } + final JsonCursor cursor = new JsonCursor(json, inputName); + final StringBuilder out = new StringBuilder(json.length()); + cursor.skipWhitespace(); + cursor.expect('{'); + out.append('{'); + cursor.skipWhitespace(); + if (cursor.peek() == '}') { + cursor.consume(); + } else { + boolean first = true; + while (true) { + cursor.skipWhitespace(); + final int keyStart = cursor.position(); + final String key = cursor.parseString(); + final String rawKey = json.substring(keyStart, cursor.position()); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if (!first) { + out.append(','); + } + first = false; + out.append(rawKey).append(':'); + switch (key) { + case "model" -> rewriteModel(cursor, out, newIdByOriginal); + case "added_tokens" -> { + cursor.skipValue(); + out.append(rewrittenAddedTokens(newIdByOriginal)); + } + case "post_processor" -> { + cursor.skipValue(); + out.append("null"); + } + default -> out.append(copyRawValue(cursor)); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a field, got '" + next + "'"); + } + } + cursor.requireEnd("Trailing content after the top-level object"); + out.append('}'); + Files.writeString(file, out.toString()); + } + + /** + * Rewrites the {@code model} object: the vocabulary renumbered to the surviving rows, the + * Unigram {@code unk_id} remapped, every other field copied byte for byte. + * + * @param cursor The cursor, positioned at the object's opening brace. + * @param out The output accumulator. + * @param newIdByOriginal The original-to-new id map. + */ + private void rewriteModel(JsonCursor cursor, StringBuilder out, + Map newIdByOriginal) { + cursor.expect('{'); + out.append('{'); + cursor.skipWhitespace(); + if (cursor.peek() == '}') { + cursor.consume(); + out.append('}'); + return; + } + boolean first = true; + while (true) { + cursor.skipWhitespace(); + final int keyStart = cursor.position(); + final String key = cursor.parseString(); + final String rawKey = json.substring(keyStart, cursor.position()); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if (!first) { + out.append(','); + } + first = false; + out.append(rawKey).append(':'); + switch (key) { + case "vocab" -> out.append(rewrittenVocab(cursor, newIdByOriginal)); + case "unk_id" -> { + cursor.skipValue(); + out.append(newIdByOriginal.getOrDefault(originalUnkId, 0)); + } + default -> out.append(copyRawValue(cursor)); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a model field, got '" + next + "'"); + } + out.append('}'); + } + + /** + * {@return the rewritten vocabulary value: for a WordPiece dictionary the kept entries with + * their new ids (raw key spans reused), for a Unigram list the kept {@code [piece, score]} + * entries byte for byte} + * + * @param cursor The cursor, positioned at the vocabulary's opening character. + * @param newIdByOriginal The original-to-new id map. + */ + private String rewrittenVocab(JsonCursor cursor, Map newIdByOriginal) { + final StringBuilder out = new StringBuilder(); + if (cursor.peek() == '{') { + cursor.consume(); + out.append('{'); + cursor.skipWhitespace(); + if (cursor.peek() == '}') { + cursor.consume(); + } else { + boolean first = true; + while (true) { + cursor.skipWhitespace(); + final int keyStart = cursor.position(); + cursor.parseString(); + final String rawKey = json.substring(keyStart, cursor.position()); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + final long originalId = cursor.parseLong(); + final Integer row = newIdByOriginal.get((int) originalId); + if (row != null) { + if (!first) { + out.append(','); + } + first = false; + out.append(rawKey).append(':').append(row); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a vocab entry, got '" + next + "'"); + } + } + out.append('}'); + } else { + cursor.expect('['); + out.append('['); + cursor.skipWhitespace(); + if (cursor.peek() == ']') { + cursor.consume(); + } else { + boolean first = true; + int originalId = 0; + while (true) { + cursor.skipWhitespace(); + final int entryStart = cursor.position(); + cursor.expect('['); + cursor.skipWhitespace(); + cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(','); + cursor.skipWhitespace(); + cursor.skipValue(); + cursor.skipWhitespace(); + cursor.expect(']'); + if (newIdByOriginal.containsKey(originalId++)) { + if (!first) { + out.append(','); + } + first = false; + out.append(json, entryStart, cursor.position()); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == ']') { + break; + } + throw cursor.malformed("Expected ',' or ']' after a vocab entry, got '" + next + "'"); + } + } + out.append(']'); + } + return out.toString(); + } + + /** + * {@return the pruned {@code added_tokens} value: the unknown and pad tokens at their new ids, + * with the flag convention Model2Vec writes (the pad token strips around itself, the unknown + * token does not)} + * + * @param newIdByOriginal The original-to-new id map. + */ + private String rewrittenAddedTokens(Map newIdByOriginal) { + record Added(int id, String content, boolean pad) { + } + final List kept = new ArrayList<>(2); + for (int id = 0; id < tokensByOriginalId.size(); id++) { + final String token = tokensByOriginalId.get(id); + final Integer row = newIdByOriginal.get(id); + if (row == null) { + continue; + } + if (token.equals(unkToken)) { + kept.add(new Added(row, token, false)); + } else if (token.equals(padToken)) { + kept.add(new Added(row, token, true)); + } + } + kept.sort(Comparator.comparingInt(Added::id)); + final StringBuilder out = new StringBuilder("["); + boolean first = true; + for (final Added added : kept) { + if (!first) { + out.append(','); + } + first = false; + out.append("{\"id\":").append(added.id()) + .append(",\"content\":").append(quoted(added.content())) + .append(",\"single_word\":").append(added.pad()) + .append(",\"lstrip\":").append(added.pad()) + .append(",\"rstrip\":").append(added.pad()) + .append(",\"normalized\":").append(added.pad()) + .append(",\"special\":true}"); + } + return out.append(']').toString(); + } + + /** + * {@return the JSON string literal for the given content, escaping the quote, the backslash, + * and control characters} + * + * @param content The string to quote. + */ + private static String quoted(String content) { + final StringBuilder out = new StringBuilder(content.length() + 2).append('"'); + for (int i = 0; i < content.length(); i++) { + final char c = content.charAt(i); + switch (c) { + case '"' -> out.append("\\\""); + case '\\' -> out.append("\\\\"); + default -> { + if (c < 0x20) { + out.append(String.format("\\u%04x", (int) c)); + } else { + out.append(c); + } + } + } + } + return out.append('"').toString(); + } + + /** + * {@return the raw text of the JSON value at the cursor, unchanged} + * + * @param cursor The cursor, positioned at the value. + */ + private String copyRawValue(JsonCursor cursor) { + final int start = cursor.position(); + cursor.skipValue(); + return json.substring(start, cursor.position()); + } + + /** The fields read out of the {@code model} object. */ + private record ModelSection(String type, List tokensById, String unkToken, Long unkId) { + } + + /** + * Parses the {@code model} object for its type, its vocabulary in id order, and its unknown + * token (by name for WordPiece, by id for Unigram). + * + * @param cursor The cursor, positioned at the object's opening brace. + * @return The parsed section. + */ + private static ModelSection parseModel(JsonCursor cursor) { + cursor.expect('{'); + cursor.skipWhitespace(); + String type = null; + List tokensById = null; + String unkToken = null; + Long unkId = null; + if (cursor.peek() == '}') { + cursor.consume(); + return new ModelSection(null, null, null, null); + } + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "type" -> type = cursor.parseString(); + case "unk_token" -> unkToken = cursor.parseString(); + case "unk_id" -> { + if (!cursor.consumeLiteral("null")) { + unkId = cursor.parseLong(); + } + } + case "vocab" -> tokensById = parseVocab(cursor); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return new ModelSection(type, tokensById, unkToken, unkId); + } + throw cursor.malformed("Expected ',' or '}' after a model field, got '" + next + "'"); + } + } + + /** + * {@return the vocabulary in id order, either from a WordPiece {@code "token": id} dictionary + * or from a Unigram {@code [piece, score]} list; dictionary ids must form a gapless range} + * + * @param cursor The cursor, positioned at the vocabulary's opening character. + */ + private static List parseVocab(JsonCursor cursor) { + if (cursor.peek() == '{') { + cursor.consume(); + cursor.skipWhitespace(); + final Map tokenById = new HashMap<>(); + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String token = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + final long id = cursor.parseLong(); + if (tokenById.putIfAbsent(id, token) != null) { + throw cursor.malformed("Vocabulary id " + id + " is assigned more than once"); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a vocab entry, got '" + next + "'"); + } + } + final List> entries = new ArrayList<>(tokenById.entrySet()); + entries.sort(Comparator.comparingLong(Map.Entry::getKey)); + final List ordered = new ArrayList<>(entries.size()); + for (int row = 0; row < entries.size(); row++) { + if (entries.get(row).getKey() != row) { + throw cursor.malformed("Vocabulary ids are not a gapless range: expected id " + row + + " but found " + entries.get(row).getKey()); + } + ordered.add(entries.get(row).getValue()); + } + return ordered; + } + cursor.expect('['); + cursor.skipWhitespace(); + final List pieces = new ArrayList<>(); + if (cursor.peek() == ']') { + cursor.consume(); + return pieces; + } + while (true) { + cursor.skipWhitespace(); + cursor.expect('['); + cursor.skipWhitespace(); + pieces.add(cursor.parseString()); + cursor.skipWhitespace(); + cursor.expect(','); + cursor.skipWhitespace(); + cursor.skipValue(); + cursor.skipWhitespace(); + cursor.expect(']'); + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == ']') { + return pieces; + } + throw cursor.malformed("Expected ',' or ']' after a vocab entry, got '" + next + "'"); + } + } + + /** + * {@return the contents of the {@code added_tokens} overlay} + * + * @param cursor The cursor, positioned at the list's opening bracket. + */ + private static Set parseAddedTokenContents(JsonCursor cursor) { + cursor.expect('['); + cursor.skipWhitespace(); + final Set contents = new HashSet<>(); + if (cursor.peek() == ']') { + cursor.consume(); + return contents; + } + while (true) { + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + String content = null; + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if ("content".equals(key)) { + content = cursor.parseString(); + } else { + cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after an added token field, got '" + next + + "'"); + } + } + if (content != null) { + contents.add(content); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == ']') { + return contents; + } + throw cursor.malformed("Expected ',' or ']' after an added token, got '" + next + "'"); + } + } + + /** The wrapper names or ids of a post-processor, plus its special-token id table. */ + private record PostProcessor(List bosNames, List eosNames, Long clsId, + Long sepId, Map specialTokenIds) { + } + + /** + * Parses the {@code post_processor} for the wrapper a single-sequence encoding adds. Supports + * the {@code TemplateProcessing} form (string or structured template) and the + * {@code BertProcessing}/{@code RobertaProcessing} forms with their {@code cls}/{@code sep} + * pairs; a {@code null} post-processor means no wrapper. + * + * @param cursor The cursor, positioned at the value. + * @return The parsed post-processor. + * @throws IllegalArgumentException Thrown if the type is not one of the supported forms. + */ + private static PostProcessor parsePostProcessor(JsonCursor cursor) { + if (cursor.consumeLiteral("null")) { + return new PostProcessor(List.of(), List.of(), null, null, Map.of()); + } + cursor.expect('{'); + cursor.skipWhitespace(); + String type = null; + List bosNames = List.of(); + List eosNames = List.of(); + Map specialTokenIds = Map.of(); + Long clsId = null; + Long sepId = null; + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "type" -> type = cursor.parseString(); + case "single" -> { + final List> wrapper = parseTemplate(cursor); + bosNames = wrapper.get(0); + eosNames = wrapper.get(1); + } + case "special_tokens" -> specialTokenIds = parseSpecialTokenIds(cursor); + case "cls" -> clsId = parseTokenIdPair(cursor); + case "sep" -> sepId = parseTokenIdPair(cursor); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a post-processor field, got '" + next + + "'"); + } + } + if (type == null) { + return new PostProcessor(List.of(), List.of(), null, null, Map.of()); + } + return switch (type) { + case "TemplateProcessing" -> + new PostProcessor(bosNames, eosNames, null, null, specialTokenIds); + case "BertProcessing", "RobertaProcessing" -> + new PostProcessor(List.of(), List.of(), clsId, sepId, specialTokenIds); + default -> throw new IllegalArgumentException("The post_processor type '" + type + + "' is not supported; expected TemplateProcessing, BertProcessing, or " + + "RobertaProcessing"); + }; + } + + /** + * {@return a two-element list: the special token names before the sequence placeholder (the + * begin-of-sequence wrapper) and those after it (the end-of-sequence wrapper); the template is + * either a string like {@code "[CLS] $A [SEP]"} or a list of {@code SpecialToken}/{@code + * Sequence} items} + * + * @param cursor The cursor, positioned at the template value. + */ + private static List> parseTemplate(JsonCursor cursor) { + final List bos = new ArrayList<>(1); + final List eos = new ArrayList<>(1); + if (cursor.peek() == '"') { + final String template = cursor.parseString(); + List current = bos; + for (final String part : template.split(" ")) { + if (part.isEmpty()) { + continue; + } + if (part.startsWith("$")) { + current = eos; + } else { + current.add(part); + } + } + return List.of(bos, eos); + } + cursor.expect('['); + cursor.skipWhitespace(); + List current = bos; + if (cursor.peek() == ']') { + cursor.consume(); + return List.of(bos, eos); + } + while (true) { + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + final String itemType = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + String id = null; + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if ("id".equals(key)) { + id = cursor.parseString(); + } else { + cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a template item field, got '" + next + + "'"); + } + } + cursor.skipWhitespace(); + cursor.expect('}'); + if ("SpecialToken".equals(itemType)) { + current.add(id); + } else if ("Sequence".equals(itemType)) { + current = eos; + } else { + throw cursor.malformed("Unknown template item type: '" + itemType + "'"); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == ']') { + break; + } + throw cursor.malformed("Expected ',' or ']' after a template item, got '" + next + "'"); + } + return List.of(bos, eos); + } + + /** + * {@return the post-processor's special-token id table, name to the first of its ids} + * + * @param cursor The cursor, positioned at the table's opening brace. + */ + private static Map parseSpecialTokenIds(JsonCursor cursor) { + cursor.expect('{'); + cursor.skipWhitespace(); + final Map ids = new HashMap<>(); + if (cursor.peek() == '}') { + cursor.consume(); + return ids; + } + while (true) { + cursor.skipWhitespace(); + final String name = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + Long id = null; + if (cursor.peek() == '}') { + cursor.consume(); + } else { + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if ("ids".equals(key)) { + cursor.expect('['); + cursor.skipWhitespace(); + id = cursor.parseLong(); + cursor.skipWhitespace(); + while (cursor.consume() == ',') { + cursor.skipWhitespace(); + cursor.skipValue(); + cursor.skipWhitespace(); + } + } else { + cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a special token field, got '" + next + + "'"); + } + } + if (id != null) { + ids.put(name, id); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return ids; + } + throw cursor.malformed("Expected ',' or '}' after a special token, got '" + next + "'"); + } + } + + /** + * {@return the id of a {@code ["token", id]} pair, as {@code cls} and {@code sep} carry it} + * + * @param cursor The cursor, positioned at the pair's opening bracket. + */ + private static Long parseTokenIdPair(JsonCursor cursor) { + cursor.expect('['); + cursor.skipWhitespace(); + cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(','); + cursor.skipWhitespace(); + final long id = cursor.parseLong(); + cursor.skipWhitespace(); + cursor.expect(']'); + return id; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java index 73b3993b1e..32b767052c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java @@ -18,10 +18,9 @@ import java.util.Arrays; - /** * Header metadata for one tensor in a safetensors file, as declared by the file's own JSON - * header. Carries no data; {@link SafetensorsFile#readFloat32(String)} resolves the bytes. + * header. Carries no data; {@link SafetensorsFile#readFloats(String)} resolves the bytes. * * @param name The tensor's name, the key it was declared under. Never {@code null}. * @param dtype The declared element type (e.g. {@code "F32"}, {@code "F16"}, @@ -65,7 +64,8 @@ public int[] shape() { } /** - * @return The number of elements the tensor holds, the product of {@link #shape()}. + * {@return the number of elements the tensor holds, the product of {@link #shape()}} + * * @throws IllegalArgumentException Thrown if the product overflows a {@code long}, which only * a crafted header can produce. */ diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java index 5fd3a332d8..50d0c42189 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java @@ -48,6 +48,7 @@ public final class CLI { final List tools = new LinkedList<>(); tools.add(new AssembleModelTool()); + tools.add(new DistillModelTool()); for (CmdLineTool tool : tools) { toolLookupMap.put(tool.getName(), tool); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java new file mode 100644 index 0000000000..2296964546 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings.cmdline; + +import opennlp.tools.cmdline.ArgumentParser.OptionalParameter; +import opennlp.tools.cmdline.ArgumentParser.ParameterDescription; + +/** + * The command-line arguments of {@link DistillModelTool}. + */ +interface DistillModelParams { + + /** + * {@return the teacher to distill: a local directory or a Hugging Face model id} + */ + @ParameterDescription(valueName = "hf-id-or-path", + description = "the sentence-transformer teacher: a Hugging Face model id (org/model) or a " + + "local directory holding tokenizer.json and onnx/model.onnx") + String getTeacher(); + + /** + * {@return the model directory to write} + */ + @ParameterDescription(valueName = "dir", + description = "the output directory for the distilled static embedding model") + String getOut(); + + /** + * {@return the number of PCA dimensions to keep} + */ + @OptionalParameter(defaultValue = "256") + @ParameterDescription(valueName = "n", + description = "the number of principal components to keep (default: 256)") + Integer getPcaDims(); +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java new file mode 100644 index 0000000000..452bfa8e63 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings.cmdline; + +import java.io.IOException; +import java.nio.file.Path; + +import opennlp.embeddings.ModelDistiller; +import opennlp.tools.cmdline.BasicCmdLineTool; +import opennlp.tools.cmdline.TerminateToolException; + +/** + * Distills a sentence-transformer teacher into a static embedding model directory, the + * {@code opennlp-embeddings DistillModel} command. This is Model2Vec's distillation + * (teacher forward pass over the vocabulary, PCA, Zipf weighting) in Java, so producing a table + * no longer needs a Python environment; see {@link ModelDistiller} for the pipeline. + * + *

The teacher is a Hugging Face model id (its files download once into a local cache) or a + * local directory holding {@code tokenizer.json} and {@code onnx/model.onnx}. A SentencePiece + * teacher also needs its trained {@code .model} file, downloaded or supplied alongside. The + * written directory is completed and verified by loading it, so a run that prints a summary is a + * directory that works.

+ */ +public class DistillModelTool extends BasicCmdLineTool { + + interface Params extends DistillModelParams { + } + + @Override + public String getShortDescription() { + return "Distills a sentence-transformer teacher into a static embedding model"; + } + + @Override + public String getHelp() { + return getBasicHelp(Params.class); + } + + @Override + public void run(String[] args) { + final Params params = validateAndParseParams(args, Params.class); + if (params.getTeacher() == null) { + throw new TerminateToolException(1, "The -teacher parameter is required: a Hugging Face " + + "model id (org/model) or a local teacher directory"); + } + if (params.getOut() == null) { + throw new TerminateToolException(1, "The -out parameter is required: the model directory " + + "to write"); + } + final ModelDistiller.ProgressListener listener = System.out::println; + final ModelDistiller.Result result; + try { + result = ModelDistiller.distill(params.getTeacher(), Path.of(params.getOut()), + params.getPcaDims(), listener); + } catch (IllegalArgumentException e) { + throw new TerminateToolException(1, e.getMessage()); + } catch (IOException e) { + throw new TerminateToolException(-1, + "IO error while distilling: " + e.getMessage(), e); + } + System.out.println("Distilled and verified a " + result.family() + " model: " + + result.vocabularySize() + " rows, " + result.teacherDimension() + "d -> " + + result.dimension() + "d, PCA kept " + + String.format("%.1f", result.explainedVarianceRatio() * 100) + "% of the variance"); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java new file mode 100644 index 0000000000..d7f117b939 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import opennlp.embeddings.StaticEmbeddingModel.Casing; +import opennlp.embeddings.StaticEmbeddingModel.Normalization; + +/** + * Fixtures shared by more than one test in this module: the small WordPiece table the geometry + * tests load, and JSON string quoting for the hand-built {@code tokenizer.json} fixtures. + */ +final class EmbeddingTestFixtures { + + /** The analogy table's tokens; the list index is the matrix row. */ + static final List ANALOGY_VOCABULARY = + List.of("[CLS]", "[SEP]", "[UNK]", "king", "queen", "man", "woman", "apple"); + + /** + * The analogy table's rows, chosen so the classic word2vec analogy is exact: + * {@code king - man + woman = [3,3] - [2,1] + [1,2] = [2,4] = queen}. The directions genuinely + * differ, so pairwise cosine similarities are not trivially 1.0. + */ + static final float[][] ANALOGY_ROWS = { + {0f, 0f}, // [CLS] + {0f, 0f}, // [SEP] + {0f, 0f}, // [UNK] + {3f, 3f}, // king + {2f, 4f}, // queen + {2f, 1f}, // man + {1f, 2f}, // woman + {-3f, -1f}, // apple: unrelated, opposite-ish direction + }; + + /** Not instantiable. */ + private EmbeddingTestFixtures() { + } + + /** + * Writes {@link #ANALOGY_VOCABULARY} and {@link #ANALOGY_ROWS} into a directory and loads them + * through the explicit WordPiece overload. + * + * @param dir The directory to write the fixture files into. + * @param normalization Whether the loaded model L2-normalizes its pooled vectors. + * @return The loaded model. + * @throws IOException Thrown if writing or reading a fixture file fails. + */ + static StaticEmbeddingModel loadAnalogyModel(Path dir, Normalization normalization) + throws IOException { + final Path vocabulary = dir.resolve("vocab.txt"); + Files.write(vocabulary, ANALOGY_VOCABULARY); + final Path safetensors = dir.resolve("model.safetensors"); + SafetensorsTestFiles.write(safetensors, + SafetensorsTestFiles.matrix("embeddings", ANALOGY_ROWS)); + return StaticEmbeddingModel.load(vocabulary, safetensors, Casing.UNCASED, normalization); + } + + /** + * {@return {@code value} as a JSON string literal, quoted and escaped} + * + * @param value The string to quote. + */ + static String jsonString(String value) { + final StringBuilder quoted = new StringBuilder("\""); + for (int i = 0; i < value.length(); i++) { + final char c = value.charAt(i); + switch (c) { + case '"' -> quoted.append("\\\""); + case '\\' -> quoted.append("\\\\"); + default -> { + if (c < 0x20) { + quoted.append(String.format("\\u%04x", (int) c)); + } else { + quoted.append(c); + } + } + } + } + return quoted.append('"').toString(); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java index c37bb51ad5..d4f728ba06 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java @@ -16,7 +16,6 @@ */ package opennlp.embeddings; -import java.io.IOException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -116,4 +115,59 @@ void testMissingFileFailsAsAnIoProblem(@TempDir Path dir) { assertThrows(IOException.class, () -> FlatJsonFields.topLevelBoolean(dir.resolve("absent.json"), "normalize")); } + + @Test + void testReadsTopLevelStrings(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"pad_token\":\"[PAD]\",\"unk_token\":\"esc\\\"aped\"}"); + + assertEquals("[PAD]", FlatJsonFields.topLevelString(file, "pad_token")); + assertEquals("esc\"aped", FlatJsonFields.topLevelString(file, "unk_token")); + } + + @Test + void testAbsentStringFieldAndExplicitNullBothReadAsNull(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"pad_token\":null}"); + + assertNull(FlatJsonFields.topLevelString(file, "pad_token")); + assertNull(FlatJsonFields.topLevelString(file, "missing")); + } + + @Test + void testNestedOccurrencesOfAStringNameDoNotMatch(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"outer\":{\"pad_token\":\"[PAD]\"}}"); + + assertNull(FlatJsonFields.topLevelString(file, "pad_token")); + } + + @Test + void testRejectsANonStringValue(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"pad_token\":true}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelString(file, "pad_token")); + assertTrue(e.getMessage().contains("must be a string")); + } + + @Test + void testRejectsADuplicateStringField(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"pad_token\":\"a\",\"pad_token\":\"b\"}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelString(file, "pad_token")); + assertTrue(e.getMessage().contains("more than once")); + } + + @Test + void testRejectsNullFileAndFieldArguments(@TempDir Path dir) throws IOException { + final Path file = write(dir, "{\"normalize\":true}"); + + assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelBoolean(null, "normalize")); + assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelBoolean(file, null)); + assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelString(null, "pad_token")); + assertThrows(IllegalArgumentException.class, + () -> FlatJsonFields.topLevelString(file, null)); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java index f4a676e724..4306307ca1 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.io.TempDir; import opennlp.embeddings.cmdline.AssembleModelTool; +import opennlp.subword.sentencepiece.SentencePieceTokenizer; import opennlp.tools.cmdline.TerminateToolException; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -148,15 +149,16 @@ void testLoadsTheRealSentencePieceModelAfterItsFileIsPresent(@TempDir Path dir) Files.writeString(dir.resolve("config.json"), "{\"normalize\":false}"); // A tokenizer.json whose vocab is the model's own poolable pieces, so the coverage check // passes; the matrix carries one row per piece. - final opennlp.subword.sentencepiece.SentencePieceTokenizer tokenizer = - opennlp.subword.sentencepiece.SentencePieceTokenizer.load(dir.resolve("sentencepiece.bpe.model")); + final SentencePieceTokenizer tokenizer = + SentencePieceTokenizer.load(dir.resolve("sentencepiece.bpe.model")); final StringBuilder vocab = new StringBuilder("{\"model\":{\"type\":\"Unigram\",\"vocab\":["); int rows = 0; for (int id = 0; id < tokenizer.vocabularySize(); id++) { if (rows > 0) { vocab.append(','); } - vocab.append('[').append(jsonString(tokenizer.idToPiece(id))).append(",-1.0]"); + vocab.append('[').append(EmbeddingTestFixtures.jsonString(tokenizer.idToPiece(id))) + .append(",-1.0]"); rows++; } vocab.append("]}}"); @@ -184,23 +186,4 @@ void testToolPrintsASummaryAndFailsLoudlyOnABadDirectory(@TempDir Path dir) thro assertTrue(e.getMessage().contains("tokenizer.json") || e.getMessage().contains("distilled"), e.getMessage()); } - - private static String jsonString(String s) { - final StringBuilder out = new StringBuilder("\""); - for (int i = 0; i < s.length(); i++) { - final char c = s.charAt(i); - switch (c) { - case '"' -> out.append("\\\""); - case '\\' -> out.append("\\\\"); - default -> { - if (c < 0x20) { - out.append(String.format("\\u%04x", (int) c)); - } else { - out.append(c); - } - } - } - } - return out.append('"').toString(); - } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java new file mode 100644 index 0000000000..90bfd99591 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * The distiller's pure pieces: the Zipf weighting matches Model2Vec's formula + * ({@code sif / (sif + p)}, {@code p} the row's share of a Zipf distribution), and the + * safetensors writer's output round-trips through the module's reader. + */ +class ModelDistillerTest { + + @Test + void testZipfWeightsFollowTheModel2vecFormula() { + // Two rows: the Zipf distribution is over 1/2 and 1/3, normalized by their sum 5/6. + final float[] weights = ModelDistiller.zipfWeights(2, 1e-4); + + assertEquals(2, weights.length); + assertEquals(1e-4 / (1e-4 + 0.6), weights[0], 1e-10); + assertEquals(1e-4 / (1e-4 + 0.4), weights[1], 1e-10); + } + + @Test + void testZipfWeightsDiscountEarlyRows() { + final float[] weights = ModelDistiller.zipfWeights(1000, 1e-4); + + // Frequent (early) tokens are down-weighted relative to rare (late) ones. + for (int i = 1; i < weights.length; i++) { + assert weights[i] > weights[i - 1]; + } + double harmonicSum = 0; + for (int j = 2; j <= 1001; j++) { + harmonicSum += 1.0 / j; + } + assertEquals(1e-4 / (1e-4 + 1.0 / 1001 / harmonicSum), weights[weights.length - 1], 1e-5); + } + + @Test + void testSafetensorsWriterRoundTripsThroughTheReader(@TempDir Path dir) throws IOException { + final float[] values = {1.5f, -2.25f, 3e8f, 0, -0.5f, 42}; + final Path file = dir.resolve("model.safetensors"); + + SafetensorsWriter.writeMatrix(file, 2, 3, values); + + final SafetensorsFile tensors = SafetensorsFile.read(file); + assertEquals(SafetensorsWriter.EMBEDDINGS_TENSOR, tensors.singleMatrixTensorName()); + assertArrayEquals(new int[] {2, 3}, tensors.tensorInfo(SafetensorsWriter.EMBEDDINGS_TENSOR) + .shape()); + assertArrayEquals(values, tensors.readFloats(SafetensorsWriter.EMBEDDINGS_TENSOR)); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java new file mode 100644 index 0000000000..9f55db8db3 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.Random; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The randomized PCA recovers the dominant subspace of a low-rank matrix: for data that is + * exactly rank-k, projecting to k components preserves the pairwise geometry (dot products) of + * the centered rows almost exactly, and it reports nearly all variance kept. A fixed seed makes + * the projection deterministic. + */ +class RandomizedPcaTest { + + private static final int ROWS = 400; + private static final int COLS = 48; + private static final int RANK = 6; + + /** + * {@return an exactly rank-{@link #RANK} matrix: a random factor times a random loading + * matrix, plus a non-zero column mean so centering is exercised} + */ + private static float[] lowRankData() { + final Random random = new Random(7); + final float[][] factors = new float[ROWS][RANK]; + final float[][] loadings = new float[RANK][COLS]; + for (final float[] row : factors) { + for (int j = 0; j < RANK; j++) { + row[j] = (float) random.nextGaussian() * (RANK - j); + } + } + for (final float[] row : loadings) { + for (int c = 0; c < COLS; c++) { + row[c] = (float) random.nextGaussian(); + } + } + final float[] data = new float[ROWS * COLS]; + for (int i = 0; i < ROWS; i++) { + for (int c = 0; c < COLS; c++) { + float value = c; // a column mean the PCA must subtract + for (int j = 0; j < RANK; j++) { + value += factors[i][j] * loadings[j][c]; + } + data[i * COLS + c] = value; + } + } + return data; + } + + private static double dot(float[] data, int rowA, int rowB, int cols) { + double dot = 0; + for (int c = 0; c < cols; c++) { + dot += (double) data[rowA * cols + c] * data[rowB * cols + c]; + } + return dot; + } + + @Test + void testRecoversTheExactSubspaceOfLowRankData() { + final float[] original = lowRankData(); + // The centered reference, for the geometry comparison. + final float[] centered = original.clone(); + for (int c = 0; c < COLS; c++) { + float mean = 0; + for (int i = 0; i < ROWS; i++) { + mean += centered[i * COLS + c]; + } + mean /= ROWS; + for (int i = 0; i < ROWS; i++) { + centered[i * COLS + c] -= mean; + } + } + + final RandomizedPca.Result result = + RandomizedPca.fitTransform(original.clone(), ROWS, COLS, RANK, 42); + + assertEquals(ROWS * RANK, result.transformed().length); + assertTrue(result.explainedVarianceRatio() > 0.999, + "rank-6 data projected to 6 components keeps (almost) all variance, got " + + result.explainedVarianceRatio()); + // Projecting exactly rank-k data onto its k principal components preserves pairwise dot + // products up to numerical noise; check the diagonal and a few off-diagonal pairs. + for (final int[] pair : new int[][] {{0, 0}, {1, 2}, {17, 399}, {5, 5}, {123, 321}}) { + final double expected = dot(centered, pair[0], pair[1], COLS); + final double actual = dot(result.transformed(), pair[0], pair[1], RANK); + final double scale = Math.max(Math.abs(expected), 1); + assertEquals(expected, actual, 1e-3 * scale, + "pairwise dot product of rows " + pair[0] + " and " + pair[1]); + } + } + + @Test + void testDeterministicForAFixedSeed() { + final float[] first = + RandomizedPca.fitTransform(lowRankData(), ROWS, COLS, RANK, 42).transformed(); + final float[] second = + RandomizedPca.fitTransform(lowRankData(), ROWS, COLS, RANK, 42).transformed(); + assertArrayEquals(first, second); + } + + @Test + void testCentersTheDataInPlace() { + final float[] data = lowRankData(); + RandomizedPca.fitTransform(data, ROWS, COLS, RANK, 42); + for (int c = 0; c < COLS; c++) { + double mean = 0; + for (int i = 0; i < ROWS; i++) { + mean += data[i * COLS + c]; + } + assertEquals(0, mean / ROWS, 1e-5, "column " + c + " is centered"); + } + } + + @Test + void testRejectsInconsistentArguments() { + final float[] data = new float[12]; + assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(null, 3, 4, 2, 42)); + assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(data, 3, 5, 2, 42)); + assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(data, 3, 4, 0, 42)); + assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(data, 3, 4, 5, 42)); + assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(data, 3, 4, 3, 42)); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java index 966de9e19b..2e25beec0c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java @@ -155,6 +155,7 @@ void testRejectsMalformedHeaders(String header) { assertTrue(e.getMessage().contains("Malformed safetensors header at offset"), () -> "Message should carry the offset, got: " + e.getMessage()); } + @Test void testSignedUnicodeEscapeFailsLoudly() { // Integer.parseInt would accept "-0FF" and decode the wrong character; the parser must not. diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java index 46ed3d33da..877555c6a6 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsTestFiles.java @@ -54,6 +54,12 @@ static Tensor matrix(String name, float[][] rows) { return new Tensor(name, new int[] {rows.length, dimension}, values); } + /** + * {@return a tensor of the given 1-D values} + * + * @param name The tensor name. + * @param values The values. + */ static Tensor vector(String name, float[] values) { return new Tensor(name, new int[] {values.length}, values); } @@ -61,6 +67,10 @@ static Tensor vector(String name, float[] values) { /** * Writes a safetensors file holding the given tensors as {@code F32}, header first, data in * declaration order. + * + * @param file The file to write. + * @param tensors The tensors, in the order they should appear in the header and data. + * @throws IOException Thrown if writing the file fails. */ static void write(Path file, Tensor... tensors) throws IOException { write(file, "F32", tensors); @@ -70,6 +80,12 @@ static void write(Path file, Tensor... tensors) throws IOException { * Writes a safetensors file encoding each tensor value as {@code dtype}, one of {@code F32}, * {@code F16} (IEEE half), or {@code BF16} (bfloat16). The {@link Tensor} values stay * {@code float}; they are converted to the target dtype's bytes here. + * + * @param file The file to write. + * @param dtype The dtype to encode every value as. + * @param tensors The tensors, in the order they should appear in the header and data. + * @throws IllegalArgumentException Thrown if {@code dtype} is not one of the three supported. + * @throws IOException Thrown if writing the file fails. */ static void write(Path file, String dtype, Tensor... tensors) throws IOException { final int elementBytes = switch (dtype) { diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java index 840ff58e0f..9cc19087d6 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelConcurrencyTest.java @@ -16,8 +16,6 @@ */ package opennlp.embeddings; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -31,7 +29,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import opennlp.embeddings.StaticEmbeddingModel.Casing; import opennlp.embeddings.StaticEmbeddingModel.Normalization; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -39,32 +36,18 @@ /** * A concurrency smoke test for the {@code @ThreadSafe} claim on {@link StaticEmbeddingModel}: * one shared instance, many threads, every concurrent result compared against the - * single-threaded reference computed up front. All operations are deterministic, so any - * deviation under concurrency is a thread-safety defect by definition: one shared instance, - * reference results computed single-threaded first, then compared under contention. + * single-threaded reference computed up front. Every operation is deterministic, so any + * deviation under contention is a thread-safety defect. */ class StaticEmbeddingModelConcurrencyTest { private static final int THREADS = 8; private static final int ITERATIONS_PER_THREAD = 200; - private static StaticEmbeddingModel loadFixture(Path dir) throws IOException { - final Path vocab = dir.resolve("vocab.txt"); - Files.write(vocab, - List.of("[CLS]", "[SEP]", "[UNK]", "king", "queen", "man", "woman", "apple")); - final float[][] rows = { - {0f, 0f}, {0f, 0f}, {0f, 0f}, - {3f, 3f}, {2f, 4f}, {2f, 1f}, {1f, 2f}, {-3f, -1f}, - }; - final Path tensors = dir.resolve("model.safetensors"); - SafetensorsTestFiles.write(tensors, SafetensorsTestFiles.matrix("embeddings", rows)); - return StaticEmbeddingModel.load(vocab, tensors, - Casing.UNCASED, Normalization.L2); - } - @Test void testConcurrentUseMatchesSingleThreadedReference(@TempDir Path dir) throws Exception { - final StaticEmbeddingModel model = loadFixture(dir); + final StaticEmbeddingModel model = + EmbeddingTestFixtures.loadAnalogyModel(dir, Normalization.L2); final float[] referenceEmbedding = model.embed("The King and Queen"); final double referenceSimilarity = model.similarity("king", "queen"); final List referenceNeighbors = model.mostSimilar("king", 3); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java index 6b3ebe1842..ba2289c085 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java @@ -131,31 +131,12 @@ private static String tokenizerJson(List pieces) { if (i > 0) { json.append(','); } - json.append('[').append(quote(pieces.get(i))).append(",-").append(i % 7).append(".5]"); + json.append('[').append(EmbeddingTestFixtures.jsonString(pieces.get(i))) + .append(",-").append(i % 7).append(".5]"); } return json.append("]}}").toString(); } - /** {@return {@code s} as a JSON string literal} */ - private static String quote(String s) { - final StringBuilder quoted = new StringBuilder("\""); - for (int i = 0; i < s.length(); i++) { - final char c = s.charAt(i); - switch (c) { - case '"' -> quoted.append("\\\""); - case '\\' -> quoted.append("\\\\"); - default -> { - if (c < 0x20) { - quoted.append(String.format("\\u%04x", (int) c)); - } else { - quoted.append(c); - } - } - } - } - return quoted.append('"').toString(); - } - @Test void testEmbedGathersRowsByPieceStringAcrossTheIdOffset(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = loadFromDirectory(writeModelDirectory(dir, null)); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java index 8a553d7c08..d07ff9481f 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java @@ -16,11 +16,7 @@ */ package opennlp.embeddings; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -38,45 +34,15 @@ /** * Exercises {@link StaticEmbeddingModel#similarity}, {@link StaticEmbeddingModel#mostSimilar}, - * and {@link StaticEmbeddingModel#analogy} against a small fixture whose vectors point in - * genuinely different directions (unlike {@link StaticEmbeddingModelTest}'s collinear rows, - * which are ideal for pooling-math assertions but would make every pairwise cosine similarity - * trivially 1.0). The fixture is built so the classic word2vec analogy has an exact answer: - * {@code king - man + woman == queen}. + * and {@link StaticEmbeddingModel#analogy} against {@link EmbeddingTestFixtures}' analogy table, + * whose vectors point in genuinely different directions (unlike {@link StaticEmbeddingModelTest}'s + * collinear rows, which are ideal for pooling-math assertions but would make every pairwise cosine + * similarity trivially 1.0). */ class StaticEmbeddingModelSimilarityTest { - private static final List VOCAB_TOKENS = - List.of("[CLS]", "[SEP]", "[UNK]", "king", "queen", "man", "woman", "apple"); - private static final int DIMENSION = 2; - - // king - man + woman = [3,3] - [2,1] + [1,2] = [2,4] = queen, exactly. - private static final float[][] ROWS = { - {0f, 0f}, // [CLS] - {0f, 0f}, // [SEP] - {0f, 0f}, // [UNK] - {3f, 3f}, // king - {2f, 4f}, // queen - {2f, 1f}, // man - {1f, 2f}, // woman - {-3f, -1f}, // apple: unrelated, opposite-ish direction - }; - - private static Path writeVocab(Path dir) throws IOException { - final Path file = dir.resolve("vocab.txt"); - Files.write(file, VOCAB_TOKENS); - return file; - } - - private static Path writeSafetensors(Path dir) throws IOException { - final Path file = dir.resolve("model.safetensors"); - SafetensorsTestFiles.write(file, SafetensorsTestFiles.matrix("embeddings", ROWS)); - return file; - } - private static StaticEmbeddingModel load(Path dir) throws IOException { - return StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir), - Casing.UNCASED, Normalization.NONE); + return EmbeddingTestFixtures.loadAnalogyModel(dir, Normalization.NONE); } @Test @@ -174,9 +140,8 @@ void testAnalogyExcludesItsOwnInputTerms(@TempDir Path dir) throws IOException { @Test void testAnalogyToleratesEqualTerms(@TempDir Path dir) throws IOException { - // A duplicate term used to crash with IllegalArgumentException("duplicate element") from - // Set.of before the exclusion moved to tokenized rows. b - a + c with a == b is just c's - // vector, so with man and woman excluded the exactly collinear queen must win. + // Repeating a term is legal: b - a + c with a == b is just c's vector, so with man and woman + // excluded the exactly collinear queen must win. final StaticEmbeddingModel model = load(dir); final List result = model.analogy("man", "man", "woman", 2); @@ -187,9 +152,8 @@ void testAnalogyToleratesEqualTerms(@TempDir Path dir) throws IOException { @Test void testAnalogyExclusionFoldsLikeEmbed(@TempDir Path dir) throws IOException { - // On an uncased model, capitalized inputs must exclude their lower-cased vocabulary rows. - // Before the fix the exclusion compared raw input strings, so "King" failed to exclude - // "king" and the analogy handed an input term back as a result. + // The exclusion folds terms through the model's own tokenizer, so on an uncased model a + // capitalized input excludes its lower-cased vocabulary row rather than handing it back. final StaticEmbeddingModel model = load(dir); final List result = model.analogy("Man", "King", "Woman", 4); @@ -207,24 +171,8 @@ void testZeroVectorRowScoresZeroNotNaN(@TempDir Path dir) throws IOException { final Path vocab = dir.resolve("zero-vocab.txt"); Files.write(vocab, List.of("[CLS]", "[SEP]", "[UNK]", "a", "zero")); final float[][] rows = {{0f, 0f}, {0f, 0f}, {0f, 0f}, {1f, 0f}, {0f, 0f}}; - final ByteBuffer buffer = ByteBuffer.allocate(rows.length * 2 * 4) - .order(ByteOrder.LITTLE_ENDIAN); - for (final float[] row : rows) { - for (final float value : row) { - buffer.putFloat(value); - } - } - final byte[] data = buffer.array(); - final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + rows.length - + ",2],\"data_offsets\":[0," + data.length + "]}}"; - final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) - .putLong(headerBytes.length).array()); - out.write(headerBytes); - out.write(data); final Path tensors = dir.resolve("zero-model.safetensors"); - Files.write(tensors, out.toByteArray()); + SafetensorsTestFiles.write(tensors, SafetensorsTestFiles.matrix("embeddings", rows)); final StaticEmbeddingModel model = StaticEmbeddingModel.load(vocab, tensors, Casing.UNCASED, Normalization.NONE); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index fc1f0c0196..9b612bbba8 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -16,11 +16,7 @@ */ package opennlp.embeddings; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -261,31 +257,10 @@ void testLoadRejectsVocabularySizeMismatch(@TempDir Path dir) throws IOException @Test void testLoadRejectsWeightsSizeMismatch(@TempDir Path dir) throws IOException { // A weights tensor sized for a different (smaller) vocabulary than the embedding matrix. - final ByteArrayOutputStream data = new ByteArrayOutputStream(); - final ByteBuffer embeddingBuffer = - ByteBuffer.allocate(ROWS.length * DIMENSION * 4).order(ByteOrder.LITTLE_ENDIAN); - for (final float[] row : ROWS) { - for (final float value : row) { - embeddingBuffer.putFloat(value); - } - } - final byte[] embeddingBytes = embeddingBuffer.array(); - data.write(embeddingBytes); - final byte[] weightBytes = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN) - .putFloat(1f).array(); - data.write(weightBytes); - final String header = "{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[" + ROWS.length + "," - + DIMENSION + "],\"data_offsets\":[0," + embeddingBytes.length + "]}," - + "\"weights\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[" - + embeddingBytes.length + "," + (embeddingBytes.length + weightBytes.length) + "]}}"; - final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - out.write(ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN) - .putLong(headerBytes.length).array()); - out.write(headerBytes); - out.write(data.toByteArray()); final Path file = dir.resolve("mismatched.safetensors"); - Files.write(file, out.toByteArray()); + SafetensorsTestFiles.write(file, + SafetensorsTestFiles.matrix("embeddings", ROWS), + SafetensorsTestFiles.vector("weights", new float[] {1f})); final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(writeVocab(dir), file, Casing.UNCASED, Normalization.NONE)); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java index dfd5ca34ab..3e14d1a6e2 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java @@ -17,14 +17,12 @@ package opennlp.embeddings; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import opennlp.embeddings.StaticEmbeddingModel.Casing; import opennlp.embeddings.StaticEmbeddingModel.Normalization; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -37,32 +35,10 @@ */ public class StaticEmbeddingUsageExampleTest { - private static final List VOCAB_TOKENS = - List.of("[CLS]", "[SEP]", "[UNK]", "king", "queen", "man", "woman", "apple"); - - // king - man + woman = [3,3] - [2,1] + [1,2] = [2,4] = queen, exactly. - private static final float[][] ROWS = { - {0f, 0f}, - {0f, 0f}, - {0f, 0f}, - {3f, 3f}, - {2f, 4f}, - {2f, 1f}, - {1f, 2f}, - {-3f, -1f}, - }; - - private static StaticEmbeddingModel load(Path dir) throws IOException { - final Path vocab = dir.resolve("vocab.txt"); - Files.write(vocab, VOCAB_TOKENS); - final Path weights = dir.resolve("model.safetensors"); - SafetensorsTestFiles.write(weights, SafetensorsTestFiles.matrix("embeddings", ROWS)); - return StaticEmbeddingModel.load(vocab, weights, Casing.UNCASED, Normalization.NONE); - } - @Test void testEmbedSimilarityNeighborsAndAnalogy(@TempDir Path dir) throws IOException { - final StaticEmbeddingModel model = load(dir); + final StaticEmbeddingModel model = + EmbeddingTestFixtures.loadAnalogyModel(dir, Normalization.NONE); final float[] vector = model.embed("king"); assertEquals(2, vector.length); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java new file mode 100644 index 0000000000..bb661fea96 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The teacher tokenizer cleaning mirrors Model2Vec: unused tokens and special added tokens other + * than the unknown and pad tokens are dropped, the survivors are renumbered in their original id + * order, and the rewritten {@code tokenizer.json} carries the pruned vocabulary, the remapped + * unknown id, a null post-processor, and only the unknown/pad added tokens. + */ +class TeacherTokenizerTest { + + // A WordPiece teacher: the special tokens are added tokens, plus one [unused] row and one + // content row pair. The post-processor wraps sequences in [CLS]/[SEP] (ids 2 and 3). + private static final String WORDPIECE_TEACHER = + "{\"version\":\"1.0\"," + + "\"normalizer\":{\"type\":\"BertNormalizer\",\"lowercase\":true}," + + "\"added_tokens\":[" + + "{\"id\":0,\"content\":\"[PAD]\",\"special\":true}," + + "{\"id\":1,\"content\":\"[UNK]\",\"special\":true}," + + "{\"id\":2,\"content\":\"[CLS]\",\"special\":true}," + + "{\"id\":3,\"content\":\"[SEP]\",\"special\":true}," + + "{\"id\":4,\"content\":\"[MASK]\",\"special\":true}]," + + "\"post_processor\":{\"type\":\"TemplateProcessing\"," + + "\"single\":[{\"SpecialToken\":{\"id\":\"[CLS]\",\"type_id\":0}}," + + "{\"Sequence\":{\"id\":\"A\",\"type_id\":0}}," + + "{\"SpecialToken\":{\"id\":\"[SEP]\",\"type_id\":0}}]," + + "\"special_tokens\":{\"[CLS]\":{\"id\":\"[CLS]\",\"ids\":[2],\"tokens\":[\"[CLS]\"]}," + + "\"[SEP]\":{\"id\":\"[SEP]\",\"ids\":[3],\"tokens\":[\"[SEP]\"]}}}," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[PAD]\":0,\"[UNK]\":1,\"[CLS]\":2,\"[SEP]\":3,\"[MASK]\":4," + + "\"hello\":5,\"[unused1]\":6,\"world\":7}}}"; + + // A Unigram teacher in the bge-m3 shape: , , , lead the vocabulary, + // trails it; all five are special added tokens. + private static final String UNIGRAM_TEACHER = + "{\"version\":\"1.0\"," + + "\"added_tokens\":[" + + "{\"id\":0,\"content\":\"\",\"special\":true}," + + "{\"id\":1,\"content\":\"\",\"special\":true}," + + "{\"id\":2,\"content\":\"\",\"special\":true}," + + "{\"id\":3,\"content\":\"\",\"special\":true}," + + "{\"id\":6,\"content\":\"\",\"special\":true}]," + + "\"post_processor\":null," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":3,\"byte_fallback\":false," + + "\"vocab\":[[\"\",0.0],[\"\",0.0],[\"\",0.0],[\"\",0.0]," + + "[\"a\",-1.5],[\"b\",-2.5],[\"\",0.0]]}}"; + + private static Path write(Path dir, String name, String content) throws IOException { + final Path file = dir.resolve(name); + Files.writeString(file, content); + return file; + } + + @Test + void testWordpieceCleaningDropsSpecialsAndUnusedTokens(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", WORDPIECE_TEACHER); + write(dir, "tokenizer_config.json", "{\"do_lower_case\":true,\"pad_token\":\"[PAD]\"}"); + + final TeacherTokenizer tokenizer = + TeacherTokenizer.read(tokenizerJson, dir.resolve("tokenizer_config.json")); + + assertEquals(TeacherTokenizer.WORDPIECE, tokenizer.modelType()); + assertEquals(4, tokenizer.vocabularySize()); + assertArrayEquals(new int[] {0, 1, 5, 7}, tokenizer.keptOriginalIds()); + assertEquals(0, tokenizer.padTokenId()); + assertEquals("[UNK]", tokenizer.unkToken()); + assertEquals("[PAD]", tokenizer.padToken()); + // Each row is fed to the teacher as [CLS, token, SEP]. + assertArrayEquals(new long[] {2, 5, 3}, tokenizer.inputSequence(2)); + } + + @Test + void testWordpieceRewriteRenumbersTheSurvivors(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", WORDPIECE_TEACHER); + write(dir, "tokenizer_config.json", "{\"pad_token\":\"[PAD]\"}"); + final TeacherTokenizer tokenizer = + TeacherTokenizer.read(tokenizerJson, dir.resolve("tokenizer_config.json")); + + final Path cleaned = dir.resolve("cleaned.json"); + tokenizer.writeCleaned(cleaned); + + // The cleaned file parses again and names exactly the surviving rows in order (the pad + // token needs its tokenizer_config to be recognized, as in the teacher). + final TeacherTokenizer reread = + TeacherTokenizer.read(cleaned, dir.resolve("tokenizer_config.json")); + assertEquals(4, reread.vocabularySize()); + assertArrayEquals(new int[] {0, 1, 2, 3}, reread.keptOriginalIds()); + final String json = Files.readString(cleaned); + assertTrue(json.contains("\"post_processor\":null"), json); + assertTrue(json.contains("\"hello\":2"), json); + assertTrue(json.contains("\"world\":3"), json); + assertTrue(!json.contains("[unused1]"), json); + assertTrue(!json.contains("[MASK]"), json); + // The unk and pad added tokens remain, with Model2Vec's flag convention. + assertTrue(json.contains("{\"id\":0,\"content\":\"[PAD]\",\"single_word\":true," + + "\"lstrip\":true,\"rstrip\":true,\"normalized\":true,\"special\":true}"), json); + assertTrue(json.contains("{\"id\":1,\"content\":\"[UNK]\",\"single_word\":false," + + "\"lstrip\":false,\"rstrip\":false,\"normalized\":false,\"special\":true}"), json); + // Untouched sections survive byte for byte. + assertTrue(json.contains("\"normalizer\":{\"type\":\"BertNormalizer\",\"lowercase\":true}"), + json); + } + + @Test + void testUnigramCleaningKeepsPadAndUnkOnly(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", UNIGRAM_TEACHER); + write(dir, "tokenizer_config.json", "{\"pad_token\":\"\"}"); + + final TeacherTokenizer tokenizer = + TeacherTokenizer.read(tokenizerJson, dir.resolve("tokenizer_config.json")); + + assertEquals(TeacherTokenizer.UNIGRAM, tokenizer.modelType()); + assertEquals(4, tokenizer.vocabularySize()); + assertArrayEquals(new int[] {1, 3, 4, 5}, tokenizer.keptOriginalIds()); + assertEquals(1, tokenizer.padTokenId()); + assertEquals("", tokenizer.unkToken()); + // No post-processor, so the input sequence is the bare token. + assertArrayEquals(new long[] {4}, tokenizer.inputSequence(2)); + } + + @Test + void testUnigramRewriteRemapsUnkIdAndKeepsScores(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", UNIGRAM_TEACHER); + write(dir, "tokenizer_config.json", "{\"pad_token\":\"\"}"); + final TeacherTokenizer tokenizer = + TeacherTokenizer.read(tokenizerJson, dir.resolve("tokenizer_config.json")); + + final Path cleaned = dir.resolve("cleaned.json"); + tokenizer.writeCleaned(cleaned); + + // The loader's own Unigram reader must see the surviving rows in order. + assertEquals(List.of("", "", "a", "b"), TokenizerJsonVocab.rows(cleaned)); + final String json = Files.readString(cleaned); + assertTrue(json.contains("\"unk_id\":1"), json); + assertTrue(json.contains("[\"a\",-1.5]"), json); + assertTrue(json.contains("\"byte_fallback\":false"), json); + } + + @Test + void testUnigramWithoutPadTokenKeepsOnlyTheUnknownToken(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", UNIGRAM_TEACHER); + + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + assertNull(tokenizer.padToken()); + assertEquals(3, tokenizer.vocabularySize()); + assertArrayEquals(new int[] {3, 4, 5}, tokenizer.keptOriginalIds()); + } + + @Test + void testRejectsAnUnsupportedModelType(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0}}}"); + + final IllegalArgumentException e = org.junit.jupiter.api.Assertions.assertThrows( + IllegalArgumentException.class, () -> TeacherTokenizer.read(tokenizerJson, null)); + assertTrue(e.getMessage().contains("BPE"), e.getMessage()); + } +} From 2501af1e5643afdd6aee070288ca47d194f9697a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 09:16:58 -0400 Subject: [PATCH 55/82] OPENNLP-1877: Fix distiller correctness bugs and cover the untested surface A review of the model distiller, which had not been reviewed before, found four defects that produce a wrong result rather than an error, and a set of classes with no tests at all. Correctness: - RandomizedPca floored the CholeskyQR jitter at an absolute value, so the decomposition was not scale invariant. Scaling the input down by 1e-6 turned an exact rank-6 recovery into an explained variance of 0.0017 with pairwise dot products wrong by three orders of magnitude, while still returning a plausible table. The jitter is now relative to the Gram trace. - RandomizedPca returned NaN instead of failing when the centered matrix was exactly zero, and a single non-finite input value poisoned its column mean and turned the whole result NaN. Both are now rejected. - The distiller guard named nanToZero tested only Float.isNaN, so an infinity from the teacher passed through into the PCA. Renamed to nonFiniteToZero and switched to Float.isFinite, with the divergence from numpy nan_to_num noted. - The PCA-skip branch kept the un-reduced matrix but left the requested dimension as the row stride, so the Zipf loop scaled the wrong cells and the writer rejected the array. Reachable with any vocabulary smaller than pcaDims, for example 100 tokens at the default 256. Tests, all previously absent: - SafetensorsWriter had no test file and its one case wrote six floats, so the 1 MiB streaming loop never crossed a chunk boundary. Added a 400x1024 round trip along with the header layout, alignment and reject paths. - ModelDistillerTest never called distill, leaving every argument check unverified. It now covers all five reject paths. - Added first tests for ModelFileNames, HuggingFaceModelCache, OnnxTeacherEncoder and the module CLI. - RandomizedPcaTest pins scale invariance, the non-finite rejection and determinism across pool sizes. Also corrected the TRAINING.md WordPiece section, which told the reader to run AssembleModel on output the distiller has already assembled. Module tests go from 156 to 273, none skipped. --- .../opennlp-embeddings/TRAINING.md | 8 +- .../embeddings/HuggingFaceModelCache.java | 117 +++++-- .../opennlp/embeddings/ModelDistiller.java | 95 ++++-- .../opennlp/embeddings/ModelFileNames.java | 18 +- .../embeddings/OnnxTeacherEncoder.java | 107 ++++-- .../opennlp/embeddings/RandomizedPca.java | 45 ++- .../opennlp/embeddings/SafetensorsWriter.java | 20 +- .../opennlp/embeddings/TeacherTokenizer.java | 14 +- .../cmdline/AssembleModelParams.java | 2 +- .../embeddings/cmdline/AssembleModelTool.java | 2 +- .../cmdline/DistillModelParams.java | 10 +- .../embeddings/cmdline/DistillModelTool.java | 12 +- .../embeddings/HuggingFaceModelCacheTest.java | 80 +++++ .../embeddings/ModelDistillerTest.java | 143 ++++++-- .../embeddings/ModelFileNamesTest.java | 86 +++++ .../embeddings/OnnxTeacherEncoderTest.java | 53 +++ .../opennlp/embeddings/RandomizedPcaTest.java | 132 +++++++- .../embeddings/SafetensorsWriterTest.java | 173 ++++++++++ .../embeddings/TeacherTokenizerTest.java | 314 +++++++++++++++++- .../opennlp/embeddings/cmdline/CLITest.java | 83 +++++ 20 files changed, 1351 insertions(+), 163 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelFileNamesTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/OnnxTeacherEncoderTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsWriterTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java diff --git a/opennlp-extensions/opennlp-embeddings/TRAINING.md b/opennlp-extensions/opennlp-embeddings/TRAINING.md index 173e88fda5..680a6e40c8 100644 --- a/opennlp-extensions/opennlp-embeddings/TRAINING.md +++ b/opennlp-extensions/opennlp-embeddings/TRAINING.md @@ -25,13 +25,15 @@ This module loads static embedding tables and, with the `DistillModel` command, opennlp-embeddings DistillModel -teacher BAAI/bge-m3 -out bge-m3-static -pcaDims 256 ``` -`-teacher` is a Hugging Face model id (its `tokenizer.json`, `tokenizer_config.json`, and `onnx/model.onnx` download once into `~/.cache/opennlp-embeddings`) or a local directory holding those files. `-pcaDims` defaults to 256. For a SentencePiece teacher like bge-m3 the trained `sentencepiece.bpe.model` is fetched alongside, because the static table keeps the teacher's segmentation. The command ends by completing the directory (the `AssembleModel` step) and verifying it by loading it, so a run that prints a summary is a directory that works. +`-teacher` is a Hugging Face model id (its `tokenizer.json`, `tokenizer_config.json`, `onnx/model.onnx`, and, for an export that splits its weights out, `onnx/model.onnx_data` download once into `~/.cache/opennlp-embeddings/-`) or a local directory holding those files. `-pcaDims` defaults to 256. For a SentencePiece teacher like bge-m3 the trained `sentencepiece.bpe.model` is fetched alongside, because the static table keeps the teacher's segmentation. The command ends by completing the directory (the `AssembleModel` step) and verifying it by loading it, so a run that prints a summary is a directory that works. + +Distil into a fresh directory. The command replaces the files it writes itself, but the assembly step never overwrites a `vocab.txt` or `tokenizer_config.json` an earlier run left behind, and a run that fails part way through leaves whatever it had written. bge-m3 is an [XLM-RoBERTa](https://arxiv.org/abs/1911.02116)/SentencePiece model with a 250k multilingual vocabulary, native dimension 1024. ### On the dimension -`pcaDims` is the one quality knob worth thinking about, and bigger is not better. Distilling bge-m3 at 256 and at 512 gives the same cross-lingual similarity within noise (English/Chinese paraphrase around 0.69 either way), while 512 doubles the matrix on disk and in memory and cuts embedding throughput. PCA to 256 already captures the useful variance of the teacher; the extra dimensions are mostly noise that dilutes the signal. 256 is a good default, and it is where the reference potion tables sit too. +`pcaDims` is the one quality knob worth thinking about, and bigger is not better. Distilling bge-m3 at 256 and at 512 gives the same cross-lingual similarity within noise (English/Chinese paraphrase around 0.69 either way), while 512 doubles the matrix on disk and in memory and cuts embedding throughput. PCA to 256 already captures the useful variance of the teacher; the extra dimensions are mostly noise that dilutes the signal. 256 is a good default, and it is where the reference Model2Vec tables (the MinishLab "potion" series) sit too. ## 2. Assemble the model directory @@ -76,7 +78,7 @@ Two tables distilled independently from the same teacher (one with this command, A WordPiece teacher (a BERT-family model such as bge-large-en) distills the same way. Its directory layout is the BERT one instead: `vocab.txt` (one token per line, line number is the row), `model.safetensors`, `config.json`, and `tokenizer_config.json` (whose `do_lower_case` sets the casing). `load` detects WordPiece from the presence of `vocab.txt`. -A distillation writes `tokenizer.json` rather than a `vocab.txt` for these, so run `AssembleModel` on the output directory: it derives `vocab.txt` from the `tokenizer.json` vocabulary in id order and `tokenizer_config.json` from the normalizer's lowercase flag. +A distillation writes `tokenizer.json` rather than a `vocab.txt`, so the two BERT files are derived: `vocab.txt` from the `tokenizer.json` vocabulary in id order, `tokenizer_config.json` from the normalizer's lowercase flag (absent, it defaults to lower-casing). `DistillModel` does this itself as its final step; `AssembleModel` is the same step run on its own, for a directory assembled by hand. ## Where a table's license comes from diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java index d9aaf36aa8..327729765a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; +import java.net.ProxySelector; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -26,6 +27,9 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; /** * Fetches the files a distillation needs from a Hugging Face model repository into a local cache @@ -35,31 +39,64 @@ */ final class HuggingFaceModelCache { - /** The hub's file-download endpoint pattern: {@code BASE}/{id}/resolve/main/{file}. */ - private static final String RESOLVE_BASE = "https://huggingface.co/"; + /** The hub's host, the prefix of every download URL. */ + private static final String HUB_BASE = "https://huggingface.co/"; - /** The ONNX graph of a hub transformer, relative to the repository root. */ - private static final String ONNX_MODEL = "onnx/model.onnx"; + /** The hub's download path between the model id and the repository-relative file name. */ + private static final String RESOLVE_PATH = "/resolve/main/"; + + /** A hub model id: an organization and a model name, both of word characters, dots, or dashes. */ + private static final Pattern MODEL_ID_PATTERN = Pattern.compile("[\\w.-]+/[\\w.-]+"); + + /** The directory the cache lives in, below the user's home directory. */ + private static final String CACHE_DIRECTORY = ".cache"; + + /** The cache's own directory, below {@link #CACHE_DIRECTORY}. */ + private static final String CACHE_NAME = "opennlp-embeddings"; + + /** The suffix of the temporary file a download streams into before it is moved into place. */ + private static final String DOWNLOAD_SUFFIX = ".download"; + + /** The HTTP status a served file answers with; anything else means the file is not there. */ + private static final int HTTP_OK = 200; + + /** How long the client waits for a connection to the hub. */ + private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(30); + + /** How long a single file download may take; an ONNX graph can be gigabytes. */ + private static final Duration DOWNLOAD_TIMEOUT = Duration.ofHours(1); /** The files a distillation needs, relative to the repository root. */ - private static final String[] REQUIRED_FILES = {"tokenizer.json", ONNX_MODEL}; + private static final List REQUIRED_FILES = + List.of(ModelFileNames.TOKENIZER_JSON, ModelFileNames.ONNX_MODEL); - /** The files used when present: the pad-token config, the SentencePiece model, and the - * external weights of an ONNX export that splits them out (as bge-m3 does). */ - private static final String[] OPTIONAL_FILES = {"tokenizer_config.json", - "sentencepiece.bpe.model", "onnx/model.onnx_data"}; + /** + * The files used when present: the pad-token config, the trained SentencePiece model under any + * of the names a repository may ship it as, and the external weights of an ONNX export that + * splits them out (as bge-m3 does). + */ + private static final List OPTIONAL_FILES = optionalFiles(); /** Not instantiable. */ private HuggingFaceModelCache() { } + /** {@return the repository-relative names of the files downloaded when the repository has them} */ + private static List optionalFiles() { + final List files = new ArrayList<>(); + files.add(ModelFileNames.TOKENIZER_CONFIG); + files.addAll(ModelFileNames.SENTENCEPIECE_MODELS); + files.add(ModelFileNames.ONNX_MODEL_DATA); + return List.copyOf(files); + } + /** * Resolves a teacher reference to a local directory holding its files. * * @param teacher A local directory, used as-is, or a Hugging Face model id * ({@code org/model}), downloaded into - * {@code ~/.cache/opennlp-embeddings/} on first use. Must not be - * {@code null}. + * {@code ~/.cache/opennlp-embeddings/org-model} on first use (the slash becomes + * a dash and dots become underscores). Must not be {@code null}. * @param listener Receives one progress line per download; may be {@code null}. * @return The local teacher directory. * @throws IllegalArgumentException Thrown if {@code teacher} is {@code null}, a local path @@ -73,15 +110,18 @@ static Path resolve(String teacher, ModelDistiller.ProgressListener listener) { if (Files.isDirectory(local)) { return local; } - if (!teacher.matches("[\\w.-]+/[\\w.-]+")) { + if (!MODEL_ID_PATTERN.matcher(teacher).matches()) { throw new IllegalArgumentException("Teacher '" + teacher + "' is neither a local " + "directory nor a Hugging Face model id (expected 'org/model')"); } - final Path cache = Path.of(System.getProperty("user.home"), ".cache", "opennlp-embeddings", - teacher.replace('/', '-').replace(".", "_")); + final Path cache = Path.of(System.getProperty("user.home"), CACHE_DIRECTORY, CACHE_NAME, + teacher.replace('/', '-').replace('.', '_')); + // A client built through the builder has no proxy selector unless one is set, so the + // http.proxyHost / https.proxyHost system properties would otherwise be ignored. final HttpClient client = HttpClient.newBuilder() .followRedirects(HttpClient.Redirect.NORMAL) - .connectTimeout(Duration.ofSeconds(30)) + .proxy(ProxySelector.getDefault()) + .connectTimeout(CONNECT_TIMEOUT) .build(); for (final String file : REQUIRED_FILES) { download(client, teacher, file, cache, true, listener); @@ -110,8 +150,8 @@ private static void download(HttpClient client, String modelId, String file, Pat return; } final HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(RESOLVE_BASE + modelId + "/resolve/main/" + file)) - .timeout(Duration.ofHours(1)) + .uri(URI.create(HUB_BASE + modelId + RESOLVE_PATH + file)) + .timeout(DOWNLOAD_TIMEOUT) .GET() .build(); final HttpResponse response; @@ -125,26 +165,47 @@ private static void download(HttpClient client, String modelId, String file, Pat throw new IllegalArgumentException("Interrupted while downloading " + file + " of " + modelId, e); } - if (response.statusCode() != 200) { - if (required) { - throw new IllegalArgumentException("Failed to download " + file + " of " + modelId - + ": HTTP " + response.statusCode() + "; the distillation needs this file"); + Path temporary = null; + try (InputStream body = response.body()) { + if (response.statusCode() != HTTP_OK) { + if (required) { + throw new IllegalArgumentException("Failed to download " + file + " of " + modelId + + ": HTTP " + response.statusCode() + "; the distillation needs this file"); + } + return; } - return; - } - try { if (listener != null) { listener.progress("Downloading " + modelId + "/" + file + " ..."); } Files.createDirectories(target.getParent()); - final Path temporary = target.resolveSibling(target.getFileName() + ".download"); - try (InputStream body = response.body()) { - Files.copy(body, temporary, StandardCopyOption.REPLACE_EXISTING); - } + // A temporary name unique per download: two processes sharing one cache directory must not + // stream two copies of the same file into one partial file and publish the interleaving. + temporary = Files.createTempFile(target.getParent(), target.getFileName().toString(), + DOWNLOAD_SUFFIX); + Files.copy(body, temporary, StandardCopyOption.REPLACE_EXISTING); Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); + temporary = null; } catch (IOException e) { throw new IllegalArgumentException("Failed to store " + file + " of " + modelId + " at " + target + ": " + e.getMessage(), e); + } finally { + deleteIfPresent(temporary); + } + } + + /** + * Deletes a partial download, if there is one, without reporting a failure to do so. + * + * @param file The file to delete; may be {@code null}. + */ + private static void deleteIfPresent(Path file) { + if (file == null) { + return; + } + try { + Files.deleteIfExists(file); + } catch (IOException e) { + // A leftover partial download costs disk space; the next attempt writes a fresh file. } } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java index c6fb15e9d5..369cd382c3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java @@ -58,14 +58,12 @@ public final class ModelDistiller { /** The fixed seed of the PCA range finder, so a distillation is reproducible. */ private static final long PCA_SEED = 42; - /** The ONNX graph inside a teacher directory. */ - private static final String ONNX_MODEL = "onnx/model.onnx"; - /** Not instantiable. */ private ModelDistiller() { } /** Receives progress messages; the command-line tool prints them. */ + @FunctionalInterface public interface ProgressListener { /** @@ -107,6 +105,7 @@ public record Result(String family, int vocabularySize, int teacherDimension, in */ public static Result distill(String teacher, Path outputDirectory, int pcaDims, ProgressListener listener) throws IOException { + checkOutput(outputDirectory, pcaDims); return distill(HuggingFaceModelCache.resolve(teacher, listener), outputDirectory, pcaDims, listener); } @@ -117,15 +116,18 @@ public static Result distill(String teacher, Path outputDirectory, int pcaDims, * @param teacherDirectory The teacher's directory, holding {@code tokenizer.json} and * {@code onnx/model.onnx}. Must not be {@code null} and must be a * directory. - * @param outputDirectory The model directory to write. Created when missing; an existing - * directory's distillation files are replaced. Must not be - * {@code null}. + * @param outputDirectory The model directory to write. Created when missing. The four files + * a distillation produces are replaced, but files a previous run's + * assembly derived ({@code vocab.txt}, {@code tokenizer_config.json}) + * are not, so distil into a fresh or emptied directory when the + * vocabulary or the dimension changes. A failure part way through + * leaves whatever was written so far. Must not be {@code null}. * @param pcaDims The number of principal components to keep; clamped to the teacher's * hidden dimension, and skipped entirely when it would not reduce a * tiny vocabulary. Model2Vec's default (and the recommended value) is * 256. - * @param listener Receives one progress line per forward-pass batch; may be - * {@code null}. + * @param listener Receives one progress line per distillation phase and one per + * forward-pass batch; may be {@code null}. * @return The distillation result, read back from the verified directory. * @throws IllegalArgumentException Thrown if an argument is {@code null} or invalid, the * teacher directory lacks its files, or the teacher cannot be run. @@ -141,23 +143,24 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc throw new IllegalArgumentException("Teacher directory does not exist or is not a " + "directory: " + teacherDirectory); } - if (outputDirectory == null) { - throw new IllegalArgumentException("OutputDirectory must not be null"); - } - if (pcaDims < 1) { - throw new IllegalArgumentException("PcaDims must be at least 1, got " + pcaDims); - } - final Path onnxFile = teacherDirectory.resolve(ONNX_MODEL); + checkOutput(outputDirectory, pcaDims); + final Path onnxFile = teacherDirectory.resolve(ModelFileNames.ONNX_MODEL); if (!Files.isRegularFile(onnxFile)) { throw new IllegalArgumentException("Teacher directory " + teacherDirectory + " has no " - + ONNX_MODEL + "; the distillation runs the teacher's ONNX export, which " + + ModelFileNames.ONNX_MODEL + "; the distillation runs the teacher's ONNX export, which " + "sentence-transformers ship on the Hugging Face hub"); } final TeacherTokenizer tokenizer = TeacherTokenizer.read( teacherDirectory.resolve(ModelFileNames.TOKENIZER_JSON), teacherDirectory.resolve(ModelFileNames.TOKENIZER_CONFIG)); final int rows = tokenizer.vocabularySize(); + if (rows < 1) { + throw new IllegalArgumentException("Teacher directory " + teacherDirectory + " has no " + + "vocabulary token left after cleaning; there is nothing to distill"); + } + report(listener, "Encoding " + rows + " vocabulary tokens of " + teacherDirectory + + " through its ONNX graph"); final float[] embeddings; final int teacherDimension; try (OnnxTeacherEncoder encoder = OnnxTeacherEncoder.load(onnxFile)) { @@ -178,24 +181,27 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc teacherDimension); } row += batchSize; - if (listener != null) { - listener.progress("Encoded " + row + " / " + rows + " vocabulary tokens"); - } + report(listener, "Encoded " + row + " / " + rows + " vocabulary tokens"); } } - nanToZero(embeddings); + nonFiniteToZero(embeddings); - final int components = Math.min(pcaDims, teacherDimension); + final int requested = Math.min(pcaDims, teacherDimension); final float[] transformed; + final int components; double explainedVarianceRatio = 1.0; - if (components >= rows) { + if (requested >= rows) { // A PCA with more components than rows is not a reduction; Model2Vec skips it with a - // warning. Only reachable for toy vocabularies. + // warning. Only reachable for toy vocabularies, which then keep the teacher's dimension. transformed = embeddings; + components = teacherDimension; } else { + report(listener, "Reducing " + rows + " x " + teacherDimension + " to " + requested + + " principal components"); final RandomizedPca.Result pca = RandomizedPca.fitTransform(embeddings, rows, - teacherDimension, components, PCA_SEED); + teacherDimension, requested, PCA_SEED); transformed = pca.transformed(); + components = requested; explainedVarianceRatio = pca.explainedVarianceRatio(); } final float[] weights = zipfWeights(rows, SIF_COEFFICIENT); @@ -207,6 +213,7 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc } } + report(listener, "Writing and verifying the model directory " + outputDirectory); Files.createDirectories(outputDirectory); SafetensorsWriter.writeMatrix(outputDirectory.resolve(ModelFileNames.SAFETENSORS), rows, components, transformed); @@ -219,6 +226,36 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc assembled.dimension(), explainedVarianceRatio); } + /** + * Validates the arguments that do not depend on the teacher, so that a distillation naming a + * hub teacher fails before it downloads anything. + * + * @param outputDirectory The model directory to write. + * @param pcaDims The number of principal components to keep. + * @throws IllegalArgumentException Thrown if the directory is {@code null} or {@code pcaDims} + * is below 1. + */ + private static void checkOutput(Path outputDirectory, int pcaDims) { + if (outputDirectory == null) { + throw new IllegalArgumentException("OutputDirectory must not be null"); + } + if (pcaDims < 1) { + throw new IllegalArgumentException("PcaDims must be at least 1, got " + pcaDims); + } + } + + /** + * Reports one progress line, if anyone is listening. + * + * @param listener The listener; may be {@code null}. + * @param message The message. + */ + private static void report(ProgressListener listener, String message) { + if (listener != null) { + listener.progress(message); + } + } + /** * {@return Model2Vec's Zipf weights: row {@code i} gets {@code sif / (sif + p_i)} with * {@code p_i = (1 / (i + 2)) / sum_j (1 / (j + 2))}, a SIF weighting under the assumption that @@ -241,14 +278,16 @@ static float[] zipfWeights(int rows, double sifCoefficient) { } /** - * Replaces NaN values with zero, Model2Vec's {@code nan_to_num} guard against a teacher - * emitting a non-finite hidden state. + * Replaces non-finite values with zero, the guard against a teacher emitting a NaN or infinite + * hidden state. Model2Vec applies numpy's {@code nan_to_num} here, which maps an infinity to the + * largest finite float; zero is used instead because an infinity of that magnitude still leaves + * the principal component analysis with nothing but that one row. * * @param values The matrix, modified in place. */ - private static void nanToZero(float[] values) { + private static void nonFiniteToZero(float[] values) { for (int i = 0; i < values.length; i++) { - if (Float.isNaN(values[i])) { + if (!Float.isFinite(values[i])) { values[i] = 0; } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java index cf7f6a4e0a..9139426cb9 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java @@ -26,6 +26,10 @@ * {@link #SAFETENSORS}, {@link #CONFIG}, {@link #VOCABULARY}, and {@link #TOKENIZER_CONFIG}; a * SentencePiece directory holds {@link #SAFETENSORS}, {@link #CONFIG}, {@link #TOKENIZER_JSON}, * and one of {@link #SENTENCEPIECE_MODELS}. + * + *

{@link #ONNX_MODEL} and {@link #ONNX_MODEL_DATA} name files of a teacher directory + * rather than of a model directory; {@link ModelDistiller} and {@link HuggingFaceModelCache} share + * them.

*/ final class ModelFileNames { @@ -48,6 +52,16 @@ final class ModelFileNames { static final List SENTENCEPIECE_MODELS = List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); + /** The ONNX graph of a teacher, relative to the teacher directory's root. */ + static final String ONNX_MODEL = "onnx/model.onnx"; + + /** The external weights an ONNX export splits out of {@link #ONNX_MODEL}, if it splits them. */ + static final String ONNX_MODEL_DATA = "onnx/model.onnx_data"; + + /** Not instantiable. */ + private ModelFileNames() { + } + /** * {@return the first of the given file names that exists as a regular file in the directory, * or {@code null} when none does} @@ -64,8 +78,4 @@ static Path firstRegularFile(Path directory, List names) { } return null; } - - /** Not instantiable. */ - private ModelFileNames() { - } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java index d90583ca69..a1df7692db 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java @@ -21,8 +21,10 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import ai.onnxruntime.NodeInfo; +import ai.onnxruntime.OnnxJavaType; import ai.onnxruntime.OnnxTensor; import ai.onnxruntime.OnnxValue; import ai.onnxruntime.OrtEnvironment; @@ -35,18 +37,32 @@ * hidden states, the forward pass Model2Vec's distillation performs per vocabulary token. The * graph is fed exactly the inputs it declares: {@code input_ids} and {@code attention_mask} for * every model, plus a zero {@code token_type_ids} for the BERT-family graphs that ask for one. - * The pooled output is the mask-weighted mean of the single rank-3 float output (the - * {@code last_hidden_state}), over the non-padding positions only. + * The pooled output is the mean of the single rank-3 float output (the + * {@code last_hidden_state}) over all of a sequence's positions; the attention mask is all ones, + * because a batch is never padded (see {@link #encodeBatch(long[][])}). * *

Not thread-safe; a distillation drives one instance from a single thread. Close it to * release the native session.

*/ final class OnnxTeacherEncoder implements AutoCloseable { + /** The id-sequence input every transformer encoder graph declares. */ + private static final String INPUT_IDS = "input_ids"; + + /** The attention-mask input every transformer encoder graph declares. */ + private static final String ATTENTION_MASK = "attention_mask"; + + /** The segment input the BERT-family graphs declare; fed all zeros. */ + private static final String TOKEN_TYPE_IDS = "token_type_ids"; + + /** The rank of the last-hidden-state output: batch, position, hidden dimension. */ + private static final int HIDDEN_STATE_RANK = 3; + private final OrtEnvironment environment; private final OrtSession session; private final boolean wantsTokenTypeIds; private final String hiddenStateOutput; + private final AtomicBoolean closed = new AtomicBoolean(); /** Holds the open session; created by {@link #load(Path)}. */ private OnnxTeacherEncoder(OrtEnvironment environment, OrtSession session, @@ -61,11 +77,12 @@ private OnnxTeacherEncoder(OrtEnvironment environment, OrtSession session, * Loads a teacher's ONNX graph. * * @param onnxFile The ONNX file. Must not be {@code null} and must exist, must declare an - * {@code input_ids} input, and must produce exactly the rank-3 float - * last-hidden-state output this encoder pools. + * {@code input_ids} input, and must produce a rank-3 float tensor output; the + * first such output is taken as the last hidden state and pooled. * @return The encoder. * @throws IllegalArgumentException Thrown if the file is missing, the graph has no - * {@code input_ids} input or no rank-3 float output, or the runtime rejects the graph. + * {@code input_ids} input or no rank-3 float tensor output, or the runtime rejects the + * graph. */ static OnnxTeacherEncoder load(Path onnxFile) { if (onnxFile == null) { @@ -75,35 +92,61 @@ static OnnxTeacherEncoder load(Path onnxFile) { throw new IllegalArgumentException("File does not exist or is not a regular file: " + onnxFile); } + final OrtEnvironment environment = OrtEnvironment.getEnvironment(); + final OrtSession session; + try (OrtSession.SessionOptions options = new OrtSession.SessionOptions()) { + session = environment.createSession(onnxFile.toString(), options); + } catch (OrtException e) { + throw new IllegalArgumentException("Failed to load ONNX graph " + onnxFile + ": " + + e.getMessage(), e); + } + // The session is open from here on, so every exit below closes it: an inspection that + // rejects the graph, or that fails outright, must not leak the native handle. try { - final OrtEnvironment environment = OrtEnvironment.getEnvironment(); - final OrtSession session = environment.createSession(onnxFile.toString(), - new OrtSession.SessionOptions()); - if (!session.getInputNames().contains("input_ids")) { - session.close(); - throw new IllegalArgumentException("ONNX graph " + onnxFile + " has no 'input_ids' " - + "input; it does not look like a transformer encoder (inputs: " + if (!session.getInputNames().contains(INPUT_IDS)) { + throw new IllegalArgumentException("ONNX graph " + onnxFile + " has no '" + INPUT_IDS + + "' input; it does not look like a transformer encoder (inputs: " + session.getInputNames() + ")"); } - final boolean wantsTokenTypeIds = session.getInputNames().contains("token_type_ids"); + final boolean wantsTokenTypeIds = session.getInputNames().contains(TOKEN_TYPE_IDS); String hiddenStateOutput = null; for (final Map.Entry output : session.getOutputInfo().entrySet()) { if (output.getValue().getInfo() instanceof TensorInfo tensorInfo - && tensorInfo.getShape().length == 3) { + && tensorInfo.type == OnnxJavaType.FLOAT + && tensorInfo.getShape().length == HIDDEN_STATE_RANK) { hiddenStateOutput = output.getKey(); break; } } if (hiddenStateOutput == null) { - session.close(); - throw new IllegalArgumentException("ONNX graph " + onnxFile + " has no rank-3 tensor " - + "output (a last hidden state) to pool (outputs: " + throw new IllegalArgumentException("ONNX graph " + onnxFile + " has no rank-3 float " + + "tensor output (a last hidden state) to pool (outputs: " + session.getOutputInfo().keySet() + ")"); } return new OnnxTeacherEncoder(environment, session, wantsTokenTypeIds, hiddenStateOutput); } catch (OrtException e) { - throw new IllegalArgumentException("Failed to load ONNX graph " + onnxFile + ": " - + e.getMessage(), e); + final IllegalArgumentException failure = new IllegalArgumentException( + "Failed to inspect ONNX graph " + onnxFile + ": " + e.getMessage(), e); + closeAfterFailure(session, failure); + throw failure; + } catch (RuntimeException e) { + closeAfterFailure(session, e); + throw e; + } + } + + /** + * Closes a session on a failing load path, reporting a close failure as a suppressed exception + * of the failure being thrown rather than in place of it. + * + * @param session The session to close. + * @param failure The exception the caller is about to throw. + */ + private static void closeAfterFailure(OrtSession session, RuntimeException failure) { + try { + session.close(); + } catch (OrtException e) { + failure.addSuppressed(e); } } @@ -137,12 +180,12 @@ float[][] encodeBatch(long[][] batch) { OnnxTensor tokenTypeIds = null; try (OnnxTensor inputIds = OnnxTensor.createTensor(environment, batch); OnnxTensor mask = OnnxTensor.createTensor(environment, attentionMask)) { - inputs.put("input_ids", inputIds); - inputs.put("attention_mask", mask); + inputs.put(INPUT_IDS, inputIds); + inputs.put(ATTENTION_MASK, mask); if (wantsTokenTypeIds) { tokenTypeIds = OnnxTensor.createTensor(environment, new long[batch.length][sequenceLength]); - inputs.put("token_type_ids", tokenTypeIds); + inputs.put(TOKEN_TYPE_IDS, tokenTypeIds); } try (OrtSession.Result result = session.run(inputs)) { final OnnxValue value = result.get(hiddenStateOutput) @@ -178,14 +221,22 @@ float[][] encodeBatch(long[][] batch) { } } - /** Closes the native session. */ + /** + * Closes the native session; calling this more than once is a no-op after the first call. + * + *

The {@link OrtEnvironment} is deliberately not closed: {@link + * OrtEnvironment#getEnvironment()} returns a process-wide singleton shared with every other + * ONNX component in the JVM, so closing it here would tear down an environment they still use. + * {@link OrtSession#close()} rejects a second call, hence the guard.

+ */ @Override public void close() { - try { - session.close(); - environment.close(); - } catch (OrtException e) { - // Closing a native resource must not mask a distillation result. + if (closed.compareAndSet(false, true)) { + try { + session.close(); + } catch (OrtException e) { + // Closing a native resource must not mask a distillation result. + } } } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java index 42352422e3..56a2b095c4 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java @@ -24,9 +24,9 @@ * Principal component analysis by randomized SVD (Halko, Martinsson, Tropp), the approximation * Model2Vec's distillation performs with a dense LAPACK SVD through scikit-learn. A dense SVD of * a vocabulary-size matrix (250k rows for a multilingual teacher) is not practical in pure Java, - * so the top components are found with a random range finder and two power iterations, which for - * the fast-decaying spectrum of transformer token embeddings recovers the same subspace as the - * exact decomposition. + * so the top components are found with a random range finder and {@value #POWER_ITERATIONS} power + * iterations, which for the fast-decaying spectrum of transformer token embeddings recovers the + * same subspace as the exact decomposition. * *

The column mean is subtracted before decomposition (the data matrix is modified in place), * and the signs of the components are fixed the way scikit-learn's full solver fixes them @@ -54,6 +54,18 @@ final class RandomizedPca { /** Jacobi eigensolver sweep cap; convergence arrives long before this. */ private static final int JACOBI_MAX_SWEEPS = 100; + /** Floor on a squared singular value, so a rank-deficient direction divides by a non-zero. */ + private static final double MIN_SQUARED_SINGULAR_VALUE = 1e-12; + + /** CholeskyQR diagonal jitter, relative to the Gram matrix's average diagonal element. */ + private static final double JITTER_RATIO = 1e-12; + + /** Factor the jitter grows by after a failed factorization. */ + private static final double JITTER_ESCALATION = 1000; + + /** Number of jitter values tried before the factorization is given up on. */ + private static final int JITTER_ATTEMPTS = 5; + /** Not instantiable. */ private RandomizedPca() { } @@ -74,7 +86,8 @@ record Result(float[] transformed, double explainedVarianceRatio) { * deterministic. * @return The projected row-major {@code rows x components} matrix and the ratio of total * variance it explains. - * @throws IllegalArgumentException Thrown if the arguments are inconsistent. + * @throws IllegalArgumentException Thrown if the arguments are inconsistent, or if the data has + * no variance to decompose (every row is identical, or a value is not finite). */ static Result fitTransform(float[] data, int rows, int cols, int components, long seed) { if (data == null) { @@ -90,6 +103,11 @@ static Result fitTransform(float[] data, int rows, int cols, int components, lon } final double[] mean = columnMean(data, rows, cols); subtractMean(data, rows, cols, mean); + final double totalVariance = totalVariance(data, rows, cols); + if (!Double.isFinite(totalVariance) || totalVariance <= 0) { + throw new IllegalArgumentException("Data has a total variance of " + totalVariance + + "; there is no subspace to find. Every row is identical, or a value is not finite."); + } final int sampleDimensions = Math.min(components + OVERSAMPLING, cols); final double[] omega = new double[cols * sampleDimensions]; final Random random = new Random(seed); @@ -126,7 +144,7 @@ static Result fitTransform(float[] data, int rows, int cols, int components, lon // mapped back through B and normalized by its singular value. final double[] componentsMajor = new double[components * cols]; for (int j = 0; j < components; j++) { - final double singularValue = Math.sqrt(Math.max(eigenvalues[j], JACOBI_EPSILON)); + final double singularValue = Math.sqrt(Math.max(eigenvalues[j], MIN_SQUARED_SINGULAR_VALUE)); for (int c = 0; c < cols; c++) { double sum = 0; for (int a = 0; a < sampleDimensions; a++) { @@ -141,7 +159,7 @@ static Result fitTransform(float[] data, int rows, int cols, int components, lon for (int j = 0; j < components; j++) { keptVariance += eigenvalues[j]; } - return new Result(transformed, keptVariance / totalVariance(data, rows, cols)); + return new Result(transformed, keptVariance / totalVariance); } /** @@ -209,16 +227,13 @@ private static double[] columnMean(float[] data, int rows, int cols) { * @param mean The per-column means. */ private static void subtractMean(float[] data, int rows, int cols, double[] mean) { - final float[] meanFloat = new float[cols]; - for (int c = 0; c < cols; c++) { - meanFloat[c] = (float) mean[c]; - } forBlocks(rows, block -> { final int start = blockStart(rows, block); final int end = blockStart(rows, block + 1); for (int i = start; i < end; i++) { for (int c = 0; c < cols; c++) { - data[i * cols + c] -= meanFloat[c]; + final int index = i * cols + c; + data[index] = (float) (data[index] - mean[c]); } } }); @@ -436,11 +451,13 @@ private static void orthonormalizeInPlace(double[] matrix, int rows, int width) for (int a = 0; a < width; a++) { trace += gram[a * width + a]; } - double jitter = Math.max(trace / width, 1) * 1e-12; + // Relative to the average diagonal element, so the factorization is unchanged when the whole + // matrix is rescaled; an absolute jitter would swamp a Gram matrix of small magnitude. + double jitter = trace / width * JITTER_RATIO; double[] lower = null; - for (int attempt = 0; attempt < 5 && lower == null; attempt++) { + for (int attempt = 0; attempt < JITTER_ATTEMPTS && lower == null; attempt++) { lower = cholesky(gram, width, jitter); - jitter *= 1000; + jitter *= JITTER_ESCALATION; } if (lower == null) { throw new IllegalStateException("Gram matrix is not positive definite even with jitter; " diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java index 4db79e6a8c..4475236932 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsWriter.java @@ -37,9 +37,18 @@ final class SafetensorsWriter { /** The name of the embedding matrix tensor, the Model2Vec convention. */ static final String EMBEDDINGS_TENSOR = "embeddings"; - // Encoding chunk, a multiple of Float.BYTES. + /** The size of the encoding buffer the matrix is streamed through; a multiple of Float.BYTES. */ private static final int WRITE_CHUNK_BYTES = 1 << 20; + /** + * The boundary the header is space-padded to, so the tensor data starts aligned. The reference + * safetensors writer pads the same way, and readers that memory-map the data section rely on it. + */ + private static final int HEADER_ALIGNMENT_BYTES = 8; + + /** The byte the header is padded with; JSON treats it as insignificant whitespace. */ + private static final byte HEADER_PADDING = ' '; + /** Not instantiable. */ private SafetensorsWriter() { } @@ -71,16 +80,21 @@ static void writeMatrix(Path file, int rows, int cols, float[] values) throws IO final String header = "{\"" + EMBEDDINGS_TENSOR + "\":{\"dtype\":\"F32\",\"shape\":[" + rows + "," + cols + "],\"data_offsets\":[0," + dataBytes + "]}}"; final byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); + final int padding = (HEADER_ALIGNMENT_BYTES + - (Long.BYTES + headerBytes.length) % HEADER_ALIGNMENT_BYTES) % HEADER_ALIGNMENT_BYTES; final Path parent = file.getParent(); if (parent != null) { Files.createDirectories(parent); } try (FileChannel channel = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { - final ByteBuffer prefix = ByteBuffer.allocate(8 + headerBytes.length) + final ByteBuffer prefix = ByteBuffer.allocate(Long.BYTES + headerBytes.length + padding) .order(ByteOrder.LITTLE_ENDIAN); - prefix.putLong(headerBytes.length); + prefix.putLong((long) headerBytes.length + padding); prefix.put(headerBytes); + for (int i = 0; i < padding; i++) { + prefix.put(HEADER_PADDING); + } prefix.flip(); writeFully(channel, prefix); final ByteBuffer chunk = ByteBuffer.allocate(WRITE_CHUNK_BYTES) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java index 15f6cb53d6..1602df3a56 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java @@ -53,6 +53,12 @@ final class TeacherTokenizer { /** Model2Vec's default token removal pattern; matched from the start, like Python re.match. */ private static final Pattern UNUSED_TOKEN_PATTERN = Pattern.compile("\\[unused\\d+\\]"); + /** Separates the items of a string post-processor template such as {@code "[CLS] $A [SEP]"}. */ + private static final Pattern TEMPLATE_ITEM_SEPARATOR = Pattern.compile("\\s+"); + + /** Marks a template item as the sequence placeholder rather than a special token. */ + private static final String SEQUENCE_PLACEHOLDER_PREFIX = "$"; + /** The WordPiece {@code model.type} of a BERT-family teacher. */ static final String WORDPIECE = "WordPiece"; @@ -301,9 +307,13 @@ long[] inputSequence(int row) { * every other field copied byte for byte from the teacher's file. * * @param file The file to write. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null}. * @throws IOException Thrown if writing fails. */ void writeCleaned(Path file) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } final Map newIdByOriginal = new HashMap<>(keptOriginalIds.length * 2); for (int row = 0; row < keptOriginalIds.length; row++) { newIdByOriginal.put(keptOriginalIds[row], row); @@ -856,11 +866,11 @@ private static List> parseTemplate(JsonCursor cursor) { if (cursor.peek() == '"') { final String template = cursor.parseString(); List current = bos; - for (final String part : template.split(" ")) { + for (final String part : TEMPLATE_ITEM_SEPARATOR.split(template)) { if (part.isEmpty()) { continue; } - if (part.startsWith("$")) { + if (part.startsWith(SEQUENCE_PLACEHOLDER_PREFIX)) { current = eos; } else { current.add(part); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java index 5f23d332d1..eb8419fd7d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelParams.java @@ -29,6 +29,6 @@ interface AssembleModelParams { * {@return the distilled model directory to assemble in place and verify} */ @ParameterDescription(valueName = "dir", - description = "the distilled model directory to complete in place and verify") + description = "The distilled model directory to complete in place and verify.") File getModelDir(); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java index 7856064897..d1a97bdda4 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java @@ -61,7 +61,7 @@ public void run(String[] args) { try { result = ModelAssembler.assemble(modelDir.toPath()); } catch (IllegalArgumentException e) { - throw new TerminateToolException(1, e.getMessage()); + throw new TerminateToolException(1, e.getMessage(), e); } catch (IOException e) { throw new TerminateToolException(-1, "IO error while assembling " + modelDir + ": " + e.getMessage(), e); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java index 2296964546..7f3f662a3a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java @@ -28,22 +28,22 @@ interface DistillModelParams { * {@return the teacher to distill: a local directory or a Hugging Face model id} */ @ParameterDescription(valueName = "hf-id-or-path", - description = "the sentence-transformer teacher: a Hugging Face model id (org/model) or a " - + "local directory holding tokenizer.json and onnx/model.onnx") + description = "The sentence-transformer teacher: a Hugging Face model id (org/model) or a " + + "local directory holding tokenizer.json and onnx/model.onnx.") String getTeacher(); /** * {@return the model directory to write} */ @ParameterDescription(valueName = "dir", - description = "the output directory for the distilled static embedding model") + description = "The output directory for the distilled static embedding model.") String getOut(); /** * {@return the number of PCA dimensions to keep} */ @OptionalParameter(defaultValue = "256") - @ParameterDescription(valueName = "n", - description = "the number of principal components to keep (default: 256)") + @ParameterDescription(valueName = "num", + description = "The number of principal components to keep, default is 256.") Integer getPcaDims(); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java index 452bfa8e63..8ea19f376d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java @@ -52,22 +52,16 @@ public String getHelp() { @Override public void run(String[] args) { + // -teacher and -out are mandatory parameters, so validateAndParseParams has already + // rejected the invocation if either is absent. final Params params = validateAndParseParams(args, Params.class); - if (params.getTeacher() == null) { - throw new TerminateToolException(1, "The -teacher parameter is required: a Hugging Face " - + "model id (org/model) or a local teacher directory"); - } - if (params.getOut() == null) { - throw new TerminateToolException(1, "The -out parameter is required: the model directory " - + "to write"); - } final ModelDistiller.ProgressListener listener = System.out::println; final ModelDistiller.Result result; try { result = ModelDistiller.distill(params.getTeacher(), Path.of(params.getOut()), params.getPcaDims(), listener); } catch (IllegalArgumentException e) { - throw new TerminateToolException(1, e.getMessage()); + throw new TerminateToolException(1, e.getMessage(), e); } catch (IOException e) { throw new TerminateToolException(-1, "IO error while distilling: " + e.getMessage(), e); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java new file mode 100644 index 0000000000..52180b837c --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The teacher-reference contract of the cache, exercised without touching the network: a local + * directory is returned as-is and anything that is neither a directory nor an {@code org/model} + * hub id is rejected before a request is made. + */ +class HuggingFaceModelCacheTest { + + @Test + void testNullTeacherFailsLoudly() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> HuggingFaceModelCache.resolve(null, null)); + assertTrue(e.getMessage().contains("must not be null"), e.getMessage()); + } + + @Test + void testLocalDirectoryIsUsedAsIs(@TempDir Path teacher) { + assertEquals(teacher, HuggingFaceModelCache.resolve(teacher.toString(), null)); + } + + @ParameterizedTest + @ValueSource(strings = {"bge-m3", "BAAI/bge m3", "BAAI/bge-m3/onnx", "/BAAI/bge-m3", + "BAAI/bge-m3/", "BAAI//bge-m3"}) + void testMalformedTeacherReferenceIsRejectedBeforeAnyRequest(String teacher) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> HuggingFaceModelCache.resolve(teacher, null)); + assertTrue(e.getMessage().contains("org/model"), e.getMessage()); + } + + /** + * A local directory wins over the hub even when its path ends in something shaped like a model + * id, so an {@code org/model} directory on disk is never downloaded over instead. + */ + @Test + void testALocalDirectoryShapedLikeAModelIdIsUsedAsIs(@TempDir Path root) throws IOException { + final Path teacher = Files.createDirectories(root.resolve("BAAI").resolve("bge-m3")); + + assertEquals(teacher, HuggingFaceModelCache.resolve(teacher.toString(), null)); + } + + /** A path that exists but is a regular file is not a teacher directory. */ + @Test + void testAnExistingRegularFileIsRejected(@TempDir Path root) throws IOException { + final Path file = Files.writeString(root.resolve("teacher.txt"), "not a directory"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> HuggingFaceModelCache.resolve(file.toString(), null)); + assertTrue(e.getMessage().contains("org/model"), e.getMessage()); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java index 90bfd99591..7ab1855d5c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java @@ -17,57 +17,154 @@ package opennlp.embeddings; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** - * The distiller's pure pieces: the Zipf weighting matches Model2Vec's formula - * ({@code sif / (sif + p)}, {@code p} the row's share of a Zipf distribution), and the - * safetensors writer's output round-trips through the module's reader. + * The distiller's argument checking and its Zipf weighting. The forward pass itself needs a real + * ONNX teacher and is exercised by the distillation script, not here, so these tests pin the + * checks that must reject a bad call before any teacher is downloaded or run. */ class ModelDistillerTest { + /** Model2Vec's SIF coefficient, the value the distiller uses. */ + private static final double SIF = 1e-4; + @Test void testZipfWeightsFollowTheModel2vecFormula() { // Two rows: the Zipf distribution is over 1/2 and 1/3, normalized by their sum 5/6. - final float[] weights = ModelDistiller.zipfWeights(2, 1e-4); + final float[] weights = ModelDistiller.zipfWeights(2, SIF); assertEquals(2, weights.length); - assertEquals(1e-4 / (1e-4 + 0.6), weights[0], 1e-10); - assertEquals(1e-4 / (1e-4 + 0.4), weights[1], 1e-10); + assertEquals(SIF / (SIF + 0.6), weights[0], 1e-10); + assertEquals(SIF / (SIF + 0.4), weights[1], 1e-10); } @Test - void testZipfWeightsDiscountEarlyRows() { - final float[] weights = ModelDistiller.zipfWeights(1000, 1e-4); + void testZipfWeightsOfASingleRowUseTheWholeDistribution() { + // One row takes all the probability mass, so p is 1 regardless of the harmonic sum. + final float[] weights = ModelDistiller.zipfWeights(1, SIF); + + assertEquals(1, weights.length); + assertEquals(SIF / (SIF + 1.0), weights[0], 1e-10); + } + + @ParameterizedTest + @ValueSource(ints = {2, 3, 100, 1000}) + void testZipfWeightsDiscountEarlyRows(int rows) { + final float[] weights = ModelDistiller.zipfWeights(rows, SIF); - // Frequent (early) tokens are down-weighted relative to rare (late) ones. - for (int i = 1; i < weights.length; i++) { - assert weights[i] > weights[i - 1]; + assertEquals(rows, weights.length); + // Frequent (early) tokens are down-weighted relative to rare (late) ones, and every weight is + // a proper fraction: sif / (sif + p) with p in (0, 1]. + for (int i = 0; i < weights.length; i++) { + assertTrue(weights[i] > 0 && weights[i] < 1, "row " + i + " has weight " + weights[i]); + if (i > 0) { + assertTrue(weights[i] > weights[i - 1], + "row " + i + " (" + weights[i] + ") must outweigh row " + (i - 1) + " (" + + weights[i - 1] + ")"); + } } + } + + @Test + void testZipfWeightsMatchTheHarmonicNormalizationOfTheLastRow() { + final int rows = 1000; + final float[] weights = ModelDistiller.zipfWeights(rows, SIF); + double harmonicSum = 0; - for (int j = 2; j <= 1001; j++) { + for (int j = 2; j <= rows + 1; j++) { harmonicSum += 1.0 / j; } - assertEquals(1e-4 / (1e-4 + 1.0 / 1001 / harmonicSum), weights[weights.length - 1], 1e-5); + assertEquals(SIF / (SIF + 1.0 / (rows + 1) / harmonicSum), weights[rows - 1], 1e-5); + } + + @Test + void testRejectsANullTeacherDirectory(@TempDir Path dir) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill((Path) null, dir, 256, null)); + assertEquals("TeacherDirectory must not be null", e.getMessage()); + } + + @Test + void testRejectsATeacherDirectoryThatIsNotADirectory(@TempDir Path dir) throws IOException { + final Path file = Files.writeString(dir.resolve("teacher"), "not a directory"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill(file, dir.resolve("out"), 256, null)); + assertTrue(e.getMessage().contains("is not a directory"), e.getMessage()); + } + + @Test + void testRejectsANullOutputDirectory(@TempDir Path dir) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill(dir, null, 256, null)); + assertEquals("OutputDirectory must not be null", e.getMessage()); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1, Integer.MIN_VALUE}) + void testRejectsANonPositivePcaDimension(int pcaDims, @TempDir Path dir) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill(dir, dir.resolve("out"), pcaDims, null)); + assertEquals("PcaDims must be at least 1, got " + pcaDims, e.getMessage()); } @Test - void testSafetensorsWriterRoundTripsThroughTheReader(@TempDir Path dir) throws IOException { - final float[] values = {1.5f, -2.25f, 3e8f, 0, -0.5f, 42}; - final Path file = dir.resolve("model.safetensors"); + void testRejectsATeacherDirectoryWithoutAnOnnxGraph(@TempDir Path dir) throws IOException { + final Path teacher = Files.createDirectory(dir.resolve("teacher")); + Files.writeString(teacher.resolve(ModelFileNames.TOKENIZER_JSON), "{}"); - SafetensorsWriter.writeMatrix(file, 2, 3, values); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill(teacher, dir.resolve("out"), 256, null)); + assertTrue(e.getMessage().contains(ModelFileNames.ONNX_MODEL), e.getMessage()); + // Nothing may be written before the teacher is known to be usable. + assertTrue(Files.notExists(dir.resolve("out")), "the output directory must not be created"); + } + + /** + * A teacher whose whole vocabulary is dropped by the cleaning has no rows to encode. That has to + * be caught before the ONNX graph is loaded, where it would otherwise surface as a zero-length + * matrix. The graph file here is a placeholder that would fail to load if it were reached. + */ + @Test + void testRejectsATeacherWhoseVocabularyIsFullyCleanedAway(@TempDir Path dir) throws IOException { + final Path teacher = Files.createDirectory(dir.resolve("teacher")); + Files.createDirectory(teacher.resolve("onnx")); + Files.writeString(teacher.resolve(ModelFileNames.ONNX_MODEL), "not a real graph"); + Files.writeString(teacher.resolve(ModelFileNames.TOKENIZER_JSON), + "{\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[unused0]\"," + + "\"vocab\":{\"[unused0]\":0}}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill(teacher, dir.resolve("out"), 256, null)); + + assertTrue(e.getMessage().contains("nothing to distill"), e.getMessage()); + assertTrue(Files.notExists(dir.resolve("out")), "the output directory must not be created"); + } - final SafetensorsFile tensors = SafetensorsFile.read(file); - assertEquals(SafetensorsWriter.EMBEDDINGS_TENSOR, tensors.singleMatrixTensorName()); - assertArrayEquals(new int[] {2, 3}, tensors.tensorInfo(SafetensorsWriter.EMBEDDINGS_TENSOR) - .shape()); - assertArrayEquals(values, tensors.readFloats(SafetensorsWriter.EMBEDDINGS_TENSOR)); + /** + * A bad output argument must be rejected before the teacher reference is resolved, so that a + * mistyped command against a hub id does not download gigabytes first. The teacher here is a + * well-formed hub id that would otherwise be fetched. + */ + @ParameterizedTest + @ValueSource(strings = {"BAAI/bge-m3", "sentence-transformers/all-MiniLM-L6-v2"}) + void testRejectsABadOutputBeforeResolvingAHubTeacher(String teacher, @TempDir Path dir) { + assertEquals("OutputDirectory must not be null", + assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill(teacher, null, 256, null)).getMessage()); + assertEquals("PcaDims must be at least 1, got 0", + assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill(teacher, dir.resolve("out"), 0, null)).getMessage()); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelFileNamesTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelFileNamesTest.java new file mode 100644 index 0000000000..5250af9c90 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelFileNamesTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * The file lookup the loader and the assembler share to find a SentencePiece model under whichever + * of its several names a teacher shipped it as: the first name that is a regular file wins, in the + * order given. + */ +class ModelFileNamesTest { + + @Test + void testReturnsTheFirstNameThatExists(@TempDir Path dir) throws IOException { + Files.writeString(dir.resolve("spiece.model"), "second"); + Files.writeString(dir.resolve("tokenizer.model"), "third"); + + assertEquals(dir.resolve("spiece.model"), + ModelFileNames.firstRegularFile(dir, ModelFileNames.SENTENCEPIECE_MODELS)); + } + + @Test + void testPrefersTheEarlierNameWhenSeveralExist(@TempDir Path dir) throws IOException { + for (final String name : ModelFileNames.SENTENCEPIECE_MODELS) { + Files.writeString(dir.resolve(name), name); + } + + assertEquals(dir.resolve(ModelFileNames.SENTENCEPIECE_MODELS.get(0)), + ModelFileNames.firstRegularFile(dir, ModelFileNames.SENTENCEPIECE_MODELS)); + } + + /** + * A directory carrying one of the names is not the model file. Accepting it would hand the + * loader a path it cannot read, one step further from the cause. + */ + @Test + void testSkipsADirectoryWithAMatchingName(@TempDir Path dir) throws IOException { + Files.createDirectory(dir.resolve("sentencepiece.bpe.model")); + Files.writeString(dir.resolve("spiece.model"), "the real one"); + + assertEquals(dir.resolve("spiece.model"), + ModelFileNames.firstRegularFile(dir, ModelFileNames.SENTENCEPIECE_MODELS)); + } + + @Test + void testReturnsNullWhenNoNameExists(@TempDir Path dir) { + assertNull(ModelFileNames.firstRegularFile(dir, ModelFileNames.SENTENCEPIECE_MODELS)); + } + + @Test + void testReturnsNullForAnEmptyNameList(@TempDir Path dir) throws IOException { + Files.writeString(dir.resolve("spiece.model"), "not asked for"); + + assertNull(ModelFileNames.firstRegularFile(dir, List.of())); + } + + @Test + void testReturnsNullForADirectoryThatDoesNotExist(@TempDir Path dir) { + assertNull(ModelFileNames.firstRegularFile(dir.resolve("missing"), + ModelFileNames.SENTENCEPIECE_MODELS)); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/OnnxTeacherEncoderTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/OnnxTeacherEncoderTest.java new file mode 100644 index 0000000000..88f71047bb --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/OnnxTeacherEncoderTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The encoder's argument contract, checked before the ONNX runtime is touched. + */ +class OnnxTeacherEncoderTest { + + @Test + void testNullFileFailsLoudly() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> OnnxTeacherEncoder.load(null)); + assertTrue(e.getMessage().contains("must not be null"), e.getMessage()); + } + + @Test + void testMissingFileFailsLoudly(@TempDir Path directory) { + final Path missing = directory.resolve("model.onnx"); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> OnnxTeacherEncoder.load(missing)); + assertTrue(e.getMessage().contains(missing.toString()), e.getMessage()); + } + + @Test + void testDirectoryIsNotARegularFile(@TempDir Path directory) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> OnnxTeacherEncoder.load(directory)); + assertTrue(e.getMessage().contains("regular file"), e.getMessage()); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java index 9f55db8db3..aaebe5bfc6 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/RandomizedPcaTest.java @@ -17,8 +17,12 @@ package opennlp.embeddings; import java.util.Random; +import java.util.concurrent.ForkJoinPool; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -37,6 +41,9 @@ class RandomizedPcaTest { private static final int COLS = 48; private static final int RANK = 6; + /** Small enough that a fixed absolute regularization would swamp the rescaled matrix. */ + private static final float SMALL_SCALE = 1e-6f; + /** * {@return an exactly rank-{@link #RANK} matrix: a random factor times a random loading * matrix, plus a non-zero column mean so centering is exercised} @@ -119,6 +126,57 @@ void testDeterministicForAFixedSeed() { assertArrayEquals(first, second); } + /** + * The decomposition is equivariant under a rescaling of the whole matrix: the projected + * coordinates scale with the input and the explained variance ratio, being a ratio, does not + * move. Any absolute (rather than relative) tolerance inside the pipeline breaks this. + */ + @Test + void testIsUnchangedByRescalingTheWholeMatrix() { + final RandomizedPca.Result unscaled = + RandomizedPca.fitTransform(lowRankData(), ROWS, COLS, RANK, 42); + final float[] scaledData = lowRankData(); + for (int i = 0; i < scaledData.length; i++) { + scaledData[i] *= SMALL_SCALE; + } + + final RandomizedPca.Result scaled = + RandomizedPca.fitTransform(scaledData, ROWS, COLS, RANK, 42); + + assertEquals(unscaled.explainedVarianceRatio(), scaled.explainedVarianceRatio(), 1e-6, + "the explained variance ratio must not depend on the magnitude of the input"); + double largest = 0; + for (final float value : unscaled.transformed()) { + largest = Math.max(largest, Math.abs(value)); + } + final double tolerance = 1e-4 * largest * SMALL_SCALE; + for (int i = 0; i < unscaled.transformed().length; i++) { + assertEquals(unscaled.transformed()[i] * (double) SMALL_SCALE, scaled.transformed()[i], + tolerance, "projected coordinate " + i); + } + } + + /** + * The parallel loops reduce per-block partial sums in a fixed block order, so the result does not + * depend on how many threads the fork/join pool runs the blocks on. + */ + @ParameterizedTest + @ValueSource(ints = {1, 2, 7}) + void testDeterministicAcrossThreadCounts(int parallelism) throws Exception { + final float[] expected = + RandomizedPca.fitTransform(lowRankData(), ROWS, COLS, RANK, 42).transformed(); + final ForkJoinPool pool = new ForkJoinPool(parallelism); + + try { + final float[] actual = pool.submit( + () -> RandomizedPca.fitTransform(lowRankData(), ROWS, COLS, RANK, 42).transformed()) + .get(); + assertArrayEquals(expected, actual); + } finally { + pool.shutdown(); + } + } + @Test void testCentersTheDataInPlace() { final float[] data = lowRankData(); @@ -133,17 +191,71 @@ void testCentersTheDataInPlace() { } @Test - void testRejectsInconsistentArguments() { + void testRejectsNullData() { + assertEquals("Data must not be null", assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(null, 3, 4, 2, 42)).getMessage()); + } + + /** + * The shape must describe the array exactly: a wrong column count, a non-positive dimension, or + * a length that is not {@code rows * cols} is rejected rather than silently reinterpreted. + */ + @ParameterizedTest + @CsvSource({"3, 5, 2", "3, 3, 2", "0, 4, 2", "3, 0, 2", "-1, 4, 2", "4, 4, 2", "2, 4, 1"}) + void testRejectsAShapeThatDoesNotDescribeTheData(int rows, int cols, int components) { final float[] data = new float[12]; + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(data, rows, cols, components, 42)); + assertTrue(e.getMessage().startsWith("Data has 12 elements, not " + rows + " x " + cols), + e.getMessage()); + } + + /** + * The component count must be a genuine reduction: at least one, no more than the column count, + * and strictly fewer than the row count (the randomized range finder has no subspace to find + * otherwise). + */ + @ParameterizedTest + @CsvSource({"0", "-1", "5", "3", "4"}) + void testRejectsAComponentCountThatIsNotAReduction(int components) { + final float[] data = new float[12]; + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(data, 3, 4, components, 42)); + assertTrue(e.getMessage().startsWith("Components must be in [1, 2], got " + components), + e.getMessage()); + } + + /** + * Data whose rows are all identical centers to exactly zero, so there is no subspace and the + * explained-variance ratio would be 0/0. That must fail loudly rather than return NaN. + */ + @Test + void testRejectsDataWithoutVariance() { + final float[] data = new float[ROWS * COLS]; + for (int i = 0; i < ROWS; i++) { + for (int c = 0; c < COLS; c++) { + data[i * COLS + c] = c; + } + } + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> RandomizedPca.fitTransform(data, ROWS, COLS, RANK, 42)); + assertTrue(e.getMessage().contains("total variance"), e.getMessage()); + } + + /** + * A non-finite value poisons the column mean, so every centered value becomes NaN and the total + * variance is NaN. The check must reject that too, not let NaN through into the table. + */ + @ParameterizedTest + @ValueSource(floats = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY}) + void testRejectsNonFiniteData(float value) { + final float[] data = lowRankData(); + data[0] = value; + assertThrows(IllegalArgumentException.class, - () -> RandomizedPca.fitTransform(null, 3, 4, 2, 42)); - assertThrows(IllegalArgumentException.class, - () -> RandomizedPca.fitTransform(data, 3, 5, 2, 42)); - assertThrows(IllegalArgumentException.class, - () -> RandomizedPca.fitTransform(data, 3, 4, 0, 42)); - assertThrows(IllegalArgumentException.class, - () -> RandomizedPca.fitTransform(data, 3, 4, 5, 42)); - assertThrows(IllegalArgumentException.class, - () -> RandomizedPca.fitTransform(data, 3, 4, 3, 42)); + () -> RandomizedPca.fitTransform(data, ROWS, COLS, RANK, 42)); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsWriterTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsWriterTest.java new file mode 100644 index 0000000000..aea8fff148 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsWriterTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The writer's output round-trips through the module's own reader, including a matrix larger than + * one encoding chunk; the bytes it lays down are the safetensors layout, header padded so the + * data starts aligned; and a shape that does not match the value count is rejected. + */ +class SafetensorsWriterTest { + + /** More floats than fit in one encoding chunk, so the streaming loop runs more than once. */ + private static final int MULTI_CHUNK_ROWS = 400; + + /** The column count of the multi-chunk fixture; rows times columns exceeds 1 MiB of floats. */ + private static final int MULTI_CHUNK_COLS = 1024; + + @Test + void testRoundTripsThroughTheReader(@TempDir Path dir) throws IOException { + final float[] values = {1.5f, -2.25f, 3e8f, 0, -0.5f, 42}; + final Path file = dir.resolve(ModelFileNames.SAFETENSORS); + + SafetensorsWriter.writeMatrix(file, 2, 3, values); + + final SafetensorsFile tensors = SafetensorsFile.read(file); + assertEquals(SafetensorsWriter.EMBEDDINGS_TENSOR, tensors.singleMatrixTensorName()); + assertArrayEquals(new int[] {2, 3}, tensors.tensorInfo(SafetensorsWriter.EMBEDDINGS_TENSOR) + .shape()); + assertArrayEquals(values, tensors.readFloats(SafetensorsWriter.EMBEDDINGS_TENSOR)); + } + + @Test + void testRoundTripsAMatrixSpanningSeveralWriteChunks(@TempDir Path dir) throws IOException { + final float[] values = new float[MULTI_CHUNK_ROWS * MULTI_CHUNK_COLS]; + for (int i = 0; i < values.length; i++) { + values[i] = i * 0.5f; + } + final Path file = dir.resolve(ModelFileNames.SAFETENSORS); + + SafetensorsWriter.writeMatrix(file, MULTI_CHUNK_ROWS, MULTI_CHUNK_COLS, values); + + final SafetensorsFile tensors = SafetensorsFile.read(file); + assertArrayEquals(new int[] {MULTI_CHUNK_ROWS, MULTI_CHUNK_COLS}, + tensors.tensorInfo(SafetensorsWriter.EMBEDDINGS_TENSOR).shape()); + assertArrayEquals(values, tensors.readFloats(SafetensorsWriter.EMBEDDINGS_TENSOR)); + } + + /** + * Pins the on-disk layout: an 8-byte little-endian header length, the JSON header, then the + * values as little-endian {@code F32}, with nothing after them. A change to any of the three + * fails here rather than in whatever tool reads the distilled model next. + */ + @Test + void testWritesTheSafetensorsByteLayout(@TempDir Path dir) throws IOException { + final float[] values = {1, -2, 0.5f, 0, 7, -0.25f}; + final Path file = dir.resolve(ModelFileNames.SAFETENSORS); + + SafetensorsWriter.writeMatrix(file, 3, 2, values); + + final byte[] bytes = Files.readAllBytes(file); + final ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + final long headerLength = buffer.getLong(); + final byte[] headerBytes = new byte[(int) headerLength]; + buffer.get(headerBytes); + assertEquals("{\"embeddings\":{\"dtype\":\"F32\",\"shape\":[3,2],\"data_offsets\":[0,24]}}", + new String(headerBytes, StandardCharsets.UTF_8).stripTrailing()); + assertEquals(Long.BYTES + headerLength + (long) values.length * Float.BYTES, bytes.length, + "the file is the length prefix, the header, and the values, with nothing after"); + for (int i = 0; i < values.length; i++) { + assertEquals(values[i], buffer.getFloat(), "value " + i + " must be little-endian F32"); + } + } + + /** + * The header is space-padded so the tensor data starts on an 8-byte boundary, the way the + * reference safetensors writer emits it. The header text length varies with the digits of the + * shape and the byte count, so every width has to be checked. + */ + @ParameterizedTest + @CsvSource({"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "99", "100", "1000"}) + void testPadsTheHeaderToAnEightByteBoundary(int cols, @TempDir Path dir) throws IOException { + final Path file = dir.resolve(ModelFileNames.SAFETENSORS); + + SafetensorsWriter.writeMatrix(file, 1, cols, new float[cols]); + + final long headerLength = ByteBuffer.wrap(Files.readAllBytes(file)) + .order(ByteOrder.LITTLE_ENDIAN).getLong(); + assertEquals(0, (Long.BYTES + headerLength) % 8, "shape [1," + cols + "] leaves the data " + + "unaligned at byte " + (Long.BYTES + headerLength)); + } + + @Test + void testCreatesTheMissingParentDirectory(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("nested").resolve("deeper") + .resolve(ModelFileNames.SAFETENSORS); + + SafetensorsWriter.writeMatrix(file, 1, 2, new float[] {1, 2}); + + assertTrue(Files.isRegularFile(file), file + " must exist"); + } + + @Test + void testReplacesAnExistingFile(@TempDir Path dir) throws IOException { + final Path file = dir.resolve(ModelFileNames.SAFETENSORS); + SafetensorsWriter.writeMatrix(file, 2, 3, new float[6]); + + SafetensorsWriter.writeMatrix(file, 1, 2, new float[] {7, 8}); + + final SafetensorsFile tensors = SafetensorsFile.read(file); + assertArrayEquals(new int[] {1, 2}, + tensors.tensorInfo(SafetensorsWriter.EMBEDDINGS_TENSOR).shape()); + assertArrayEquals(new float[] {7, 8}, + tensors.readFloats(SafetensorsWriter.EMBEDDINGS_TENSOR)); + } + + @Test + void testRejectsANullFile() { + assertEquals("File must not be null", assertThrows(IllegalArgumentException.class, + () -> SafetensorsWriter.writeMatrix(null, 1, 1, new float[1])).getMessage()); + } + + @Test + void testRejectsNullValues(@TempDir Path dir) { + assertEquals("Values must not be null", assertThrows(IllegalArgumentException.class, + () -> SafetensorsWriter.writeMatrix(dir.resolve(ModelFileNames.SAFETENSORS), 1, 1, null)) + .getMessage()); + } + + @ParameterizedTest + @CsvSource({"0, 2, 2", "2, 0, 2", "-1, 2, 2", "2, 3, 5", "2, 3, 7", "1, 1, 0"}) + void testRejectsAShapeThatDoesNotMatchTheValues(int rows, int cols, int valueCount, + @TempDir Path dir) { + final Path file = dir.resolve(ModelFileNames.SAFETENSORS); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> SafetensorsWriter.writeMatrix(file, rows, cols, new float[valueCount])); + + assertEquals("Values has " + valueCount + " elements, not " + rows + " x " + cols, + e.getMessage()); + assertTrue(Files.notExists(file), "a rejected write must not leave a file behind"); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java index bb661fea96..ef00de07fe 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java @@ -23,10 +23,15 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -117,8 +122,8 @@ void testWordpieceRewriteRenumbersTheSurvivors(@TempDir Path dir) throws IOExcep assertTrue(json.contains("\"post_processor\":null"), json); assertTrue(json.contains("\"hello\":2"), json); assertTrue(json.contains("\"world\":3"), json); - assertTrue(!json.contains("[unused1]"), json); - assertTrue(!json.contains("[MASK]"), json); + assertFalse(json.contains("[unused1]"), json); + assertFalse(json.contains("[MASK]"), json); // The unk and pad added tokens remain, with Model2Vec's flag convention. assertTrue(json.contains("{\"id\":0,\"content\":\"[PAD]\",\"single_word\":true," + "\"lstrip\":true,\"rstrip\":true,\"normalized\":true,\"special\":true}"), json); @@ -180,8 +185,309 @@ void testRejectsAnUnsupportedModelType(@TempDir Path dir) throws IOException { final Path tokenizerJson = write(dir, "tokenizer.json", "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0}}}"); - final IllegalArgumentException e = org.junit.jupiter.api.Assertions.assertThrows( - IllegalArgumentException.class, () -> TeacherTokenizer.read(tokenizerJson, null)); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TeacherTokenizer.read(tokenizerJson, null)); assertTrue(e.getMessage().contains("BPE"), e.getMessage()); } + + @Test + void testRejectsANullTokenizerJsonFile() { + assertEquals("TokenizerJsonFile must not be null", assertThrows( + IllegalArgumentException.class, () -> TeacherTokenizer.read(null, null)).getMessage()); + } + + @Test + void testRejectsAMissingTokenizerJsonFile(@TempDir Path dir) { + final Path missing = dir.resolve("tokenizer.json"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TeacherTokenizer.read(missing, null)); + assertEquals("File does not exist or is not a regular file: " + missing, e.getMessage()); + } + + /** + * The teacher must be rejected, not half-read, when it cannot describe a distilled table. Each + * case names the part of the contract it breaks and a fragment the message has to carry, so a + * silently weakened check shows up as a failing row rather than as a corrupt model directory. + */ + @ParameterizedTest + @CsvSource(delimiter = ';', value = { + "no model at all;{\"version\":\"1.0\"};has no model with a vocabulary", + "a model without a vocabulary;{\"model\":{\"type\":\"WordPiece\"}};" + + "has no model with a vocabulary", + "no unknown token;{\"model\":{\"type\":\"WordPiece\",\"vocab\":{\"a\":0}}};" + + "does not name an unknown token", + "an unknown token outside the vocabulary;{\"model\":{\"type\":\"WordPiece\"," + + "\"unk_token\":\"[UNK]\",\"vocab\":{\"a\":0}}};it is not in the vocabulary", + "a Unigram unk_id out of range;{\"model\":{\"type\":\"Unigram\",\"unk_id\":9," + + "\"vocab\":[[\"a\",0.0]]}};does not name an unknown token", + "vocabulary ids with a gap;{\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"a\"," + + "\"vocab\":{\"a\":0,\"b\":2}}};not a gapless range", + "an unsupported post-processor;{\"post_processor\":{\"type\":\"ByteLevel\"}," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"a\",\"vocab\":{\"a\":0}}};" + + "is not supported", + "trailing content;{\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"a\"," + + "\"vocab\":{\"a\":0}}} junk;Trailing content"}) + void testRejectsATeacherItCannotDistill(String reason, String teacherJson, String messagePart, + @TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", teacherJson); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TeacherTokenizer.read(tokenizerJson, null), reason); + assertTrue(e.getMessage().contains(messagePart), + "a teacher with " + reason + " reported: " + e.getMessage()); + } + + /** + * The other shape a {@code TemplateProcessing} template takes: a single string, whose items are + * separated by whitespace rather than being a list of objects. + */ + @Test + void testReadsAStringTemplatePostProcessor(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"post_processor\":{\"type\":\"TemplateProcessing\"," + + "\"single\":\" $A \"," + + "\"special_tokens\":{\"\":{\"id\":\"\",\"ids\":[0]}," + + "\"\":{\"id\":\"\",\"ids\":[2]}}}," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"\"," + + "\"vocab\":{\"\":0,\"\":1,\"\":2,\"\":3,\"a\":4}}}"); + + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + // Row 4 is 'a'; the string template wraps it the same way the structured form would. + assertArrayEquals(new long[] {0, 4, 2}, tokenizer.inputSequence(4)); + } + + /** + * A {@code BertProcessing} post-processor carries its wrapper as {@code cls}/{@code sep} token + * pairs instead of as a template, and the ids come straight from those pairs. + */ + @Test + void testReadsABertProcessingPostProcessor(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"post_processor\":{\"type\":\"BertProcessing\"," + + "\"cls\":[\"[CLS]\",2],\"sep\":[\"[SEP]\",3]}," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[PAD]\":0,\"[UNK]\":1,\"[CLS]\":2,\"[SEP]\":3,\"hello\":4}}}"); + + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + assertEquals(5, tokenizer.vocabularySize()); + assertArrayEquals(new long[] {2, 4, 3}, tokenizer.inputSequence(4)); + } + + /** A teacher without a post-processor feeds the bare token, with no wrapper ids. */ + @Test + void testANullPostProcessorAddsNoWrapper(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"post_processor\":null,\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[UNK]\":0,\"hello\":1}}}"); + + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + assertArrayEquals(new long[] {1}, tokenizer.inputSequence(1)); + } + + /** + * A pad token the teacher's {@code tokenizer_config.json} names but the vocabulary does not have + * is not a row; it must not be kept, and the pad id falls back to 0. + */ + @Test + void testAPadTokenOutsideTheVocabularyIsIgnored(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", UNIGRAM_TEACHER); + write(dir, "tokenizer_config.json", "{\"pad_token\":\"\"}"); + + final TeacherTokenizer tokenizer = + TeacherTokenizer.read(tokenizerJson, dir.resolve("tokenizer_config.json")); + + assertEquals(0, tokenizer.padTokenId()); + assertArrayEquals(new int[] {3, 4, 5}, tokenizer.keptOriginalIds()); + } + + /** + * A template that names its special tokens without carrying a {@code special_tokens} table has to + * resolve those names through the vocabulary instead. + */ + @Test + void testATemplateWithoutASpecialTokenTableResolvesThroughTheVocabulary(@TempDir Path dir) + throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"post_processor\":{\"type\":\"TemplateProcessing\"," + + "\"single\":[{\"SpecialToken\":{\"id\":\"[CLS]\",\"type_id\":0}}," + + "{\"Sequence\":{\"id\":\"A\",\"type_id\":0}}," + + "{\"SpecialToken\":{\"id\":\"[SEP]\",\"type_id\":0}}]}," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[PAD]\":0,\"[UNK]\":1,\"[CLS]\":2,\"[SEP]\":3,\"hello\":4}}}"); + + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + assertArrayEquals(new long[] {2, 4, 3}, tokenizer.inputSequence(4)); + } + + @Test + void testRejectsATemplateSpecialTokenThatResolvesNowhere(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"post_processor\":{\"type\":\"TemplateProcessing\"," + + "\"single\":[{\"SpecialToken\":{\"id\":\"[BOS]\",\"type_id\":0}}," + + "{\"Sequence\":{\"id\":\"A\",\"type_id\":0}}]}," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[UNK]\":0,\"hello\":1}}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TeacherTokenizer.read(tokenizerJson, null)); + assertTrue(e.getMessage().contains("[BOS]"), e.getMessage()); + } + + @Test + void testRejectsAVocabularyIdUsedTwice(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"a\",\"vocab\":{\"a\":0,\"b\":0}}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TeacherTokenizer.read(tokenizerJson, null)); + assertTrue(e.getMessage().contains("assigned more than once"), e.getMessage()); + } + + @Test + void testAnExplicitlyNullUnkIdCountsAsNoUnknownToken(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"model\":{\"type\":\"Unigram\",\"unk_id\":null,\"vocab\":[[\"a\",0.0]]}}"); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TeacherTokenizer.read(tokenizerJson, null)); + assertTrue(e.getMessage().contains("does not name an unknown token"), e.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"", " \n\t "}) + void testRejectsAnEmptyTokenizerJson(String content, @TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", content); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TeacherTokenizer.read(tokenizerJson, null)); + assertTrue(e.getMessage().contains("Unexpected end of input"), e.getMessage()); + } + + /** + * The removal pattern is matched from the start of the token, so it drops a token that begins + * with an {@code [unusedN]} marker and keeps everything else, including a marker without digits + * and one that is not at the start. + */ + @ParameterizedTest + @CsvSource(delimiter = ';', value = { + "[unused0];1", + "[unused12];1", + "[unused7]tail;1", + "[unused];2", + "[unusedx];2", + "x[unused1];2", + "[UNUSED1];2"}) + void testUnusedTokenRemovalMatchesFromTheStartOnly(String token, int expectedSize, + @TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[UNK]\":0,\"" + token + "\":1}}}"); + + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + assertEquals(expectedSize, tokenizer.vocabularySize()); + } + + /** + * Vocabulary entries are copied as raw spans, so a teacher's escapes reach the distilled file + * untouched and still decode to the pieces the loader resolves matrix rows by. + */ + @Test + void testUnicodeVocabularyEntriesSurviveTheRewrite(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"version\":\"1.0\",\"post_processor\":null," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":0,\"vocab\":[[\"\",0.0]," + + "[\"caf\\u00e9\",-1.0],[\"e\\u0301\",-2.0],[\"\\ud83d\\ude00\",-3.0]," + + "[\"a\",-4.0]]}}"); + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + final Path cleaned = dir.resolve("cleaned.json"); + tokenizer.writeCleaned(cleaned); + + assertEquals(5, tokenizer.vocabularySize()); + final String json = Files.readString(cleaned); + assertTrue(json.contains("[\"caf\\u00e9\",-1.0]"), json); + assertTrue(json.contains("[\"e\\u0301\",-2.0]"), json); + assertTrue(json.contains("[\"\\ud83d\\ude00\",-3.0]"), json); + // A precomposed letter, a base letter plus a combining acute, and a supplementary-plane + // character all decode to what the teacher declared. + assertEquals(List.of("", "caf\u00e9", "e\u0301", "\uD83D\uDE00", "a"), + TokenizerJsonVocab.rows(cleaned)); + } + + /** + * The added-token overlay is the only part of the rewrite that re-encodes a token string rather + * than copying its raw span, so it has to escape what JSON requires. + */ + @Test + void testTheAddedTokenOverlayEscapesTheUnknownTokenContent(@TempDir Path dir) throws IOException { + // The unknown token carries a backslash and a tab. + final String rawToken = "\"\""; + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"version\":\"1.0\"," + + "\"added_tokens\":[{\"id\":0,\"content\":" + rawToken + ",\"special\":true}]," + + "\"post_processor\":null," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":0," + + "\"vocab\":[[" + rawToken + ",0.0],[\"a\",-1.0]]}}"); + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + final Path cleaned = dir.resolve("cleaned.json"); + tokenizer.writeCleaned(cleaned); + + assertEquals(2, tokenizer.vocabularySize()); + final String json = Files.readString(cleaned); + assertTrue(json.contains("[" + rawToken + ",0.0]"), json); + assertTrue(json.contains("\"content\":\"\""), json); + } + + /** The rewrite emits only fields the teacher had, so an absent overlay stays absent. */ + @Test + void testATeacherWithoutAnAddedTokensSectionWritesNoOverlay(@TempDir Path dir) + throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"version\":\"1.0\",\"post_processor\":null," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":0," + + "\"vocab\":[[\"\",0.0],[\"a\",-1.0]]}}"); + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + final Path cleaned = dir.resolve("cleaned.json"); + tokenizer.writeCleaned(cleaned); + + assertFalse(Files.readString(cleaned).contains("added_tokens")); + assertEquals(List.of("", "a"), TokenizerJsonVocab.rows(cleaned)); + } + + /** + * The overlay is pruned by token content alone: the {@code special} flag is never read, so a + * plain vocabulary extension is dropped from the distilled table just like {@code [MASK]} is. + */ + @Test + void testEveryAddedTokenIsDroppedRegardlessOfItsSpecialFlag(@TempDir Path dir) + throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", + "{\"version\":\"1.0\"," + + "\"added_tokens\":[{\"id\":1,\"content\":\"[UNK]\",\"special\":true}," + + "{\"id\":2,\"content\":\"covid\",\"special\":false}]," + + "\"post_processor\":null," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"hello\":0,\"[UNK]\":1,\"covid\":2}}}"); + + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + assertArrayEquals(new int[] {0, 1}, tokenizer.keptOriginalIds()); + } + + @Test + void testWriteCleanedRejectsANullFile(@TempDir Path dir) throws IOException { + final Path tokenizerJson = write(dir, "tokenizer.json", UNIGRAM_TEACHER); + final TeacherTokenizer tokenizer = TeacherTokenizer.read(tokenizerJson, null); + + assertEquals("File must not be null", assertThrows( + IllegalArgumentException.class, () -> tokenizer.writeCleaned(null)).getMessage()); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java new file mode 100644 index 0000000000..f69c2b4bf3 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings.cmdline; + +import java.util.Set; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import opennlp.tools.cmdline.BasicCmdLineTool; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The command names the dispatcher offers and the help every tool produces. The names are the + * module's public surface (TRAINING.md and the manual quote them), so a class rename that changes + * a command has to fail here rather than in a user's shell. + */ +class CLITest { + + /** {@return the tools the dispatcher registers, as parameterized-test arguments} */ + static Stream tools() { + return Stream.of(new AssembleModelTool(), new DistillModelTool()); + } + + @Test + void testOffersExactlyTheDistillationCommands() { + assertEquals(Set.of("AssembleModel", "DistillModel"), CLI.getToolNames()); + } + + @Test + void testTheToolNamesCannotBeModifiedByACaller() { + final Set names = CLI.getToolNames(); + + assertThrows(UnsupportedOperationException.class, () -> names.add("Other")); + } + + @ParameterizedTest + @MethodSource("tools") + void testEveryRegisteredToolDescribesItself(BasicCmdLineTool tool) { + assertTrue(CLI.getToolNames().contains(tool.getName()), + tool.getName() + " must be registered with the dispatcher"); + assertFalse(tool.getShortDescription().isBlank(), + tool.getName() + " must have a short description for the usage listing"); + assertTrue(tool.getHelp().contains(tool.getName()), tool.getHelp()); + } + + @Test + void testDistillHelpNamesEveryParameter() { + final String help = new DistillModelTool().getHelp(); + + assertTrue(help.contains("-teacher hf-id-or-path"), help); + assertTrue(help.contains("-out dir"), help); + // The optional parameter is bracketed, so a user can see it may be omitted. + assertTrue(help.contains("[-pcaDims "), help); + } + + @Test + void testAssembleHelpNamesItsParameter() { + final String help = new AssembleModelTool().getHelp(); + + assertTrue(help.contains("-modelDir dir"), help); + } +} From 1bc19282f8e835fd394c5a3256c033619dab7ba1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 28 Jul 2026 10:37:40 -0400 Subject: [PATCH 56/82] OPENNLP-1877: Pin and verify teacher downloads The distiller fetched teacher weights from a moving ref and executed the ONNX graph without checking anything. DownloadUtil has always refused a model whose sha512 sidecar it cannot read, so this brings the hub path to the same posture. - Resolve the ref to a commit once, then request every file at that sha, so a force-push midway cannot mix two revisions into one cache directory. A teacher may now name a revision as org/model@revision. - Verify every file against the digest the hub publishes in x-linked-etag, choosing the algorithm by hex length: 40 is the git blob SHA-1 over "blob \0" and the bytes, 64 is the SHA-256 of the content. The digest is computed over what was written to disk. A mismatch deletes the partial file and throws, in DownloadUtil's wording. - Refuse a file whose digest the hub does not publish, rather than accept something unverifiable. This is the point of the change. - Record the resolved commit in .opennlp-revision, which also marks the directory as a complete snapshot, and carry it into the distilled model's config.json as teacher_revision so a table can name the teacher it came from. - Make the cache directory name injective. It replaced '/', '.' and '@' without distinguishing them, so acme/model@v1, acme/model.v1 and acme/model_v1 shared one directory, and the cached path answers without contacting the hub, which would have served one teacher another's files. Headers are read by walking HttpResponse.previousResponse(): with Redirect.NORMAL the final CDN response carries neither header, while the redirecting hub response carries both. A test fails if that walk is removed. Tests need no network. A loopback HttpServer serves canned replies and records the requests made, covering both digest forms, a corrupted body, a missing and six malformed etags, optional and required 404s, the redirect path, the zero request cached path, revision pinning and the directory collision. Module tests go from 273 to 310, none skipped. --- opennlp-extensions/opennlp-embeddings/pom.xml | 18 + .../embeddings/HuggingFaceModelCache.java | 516 ++++++++++-- .../opennlp/embeddings/ModelDistiller.java | 25 +- .../cmdline/DistillModelParams.java | 5 +- .../embeddings/cmdline/DistillModelTool.java | 5 +- .../embeddings/HuggingFaceModelCacheTest.java | 769 +++++++++++++++++- 6 files changed, 1268 insertions(+), 70 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/pom.xml b/opennlp-extensions/opennlp-embeddings/pom.xml index 27f892340e..48a007ed5b 100644 --- a/opennlp-extensions/opennlp-embeddings/pom.xml +++ b/opennlp-extensions/opennlp-embeddings/pom.xml @@ -78,6 +78,24 @@ + + + + de.thetaphi + forbiddenapis + + + + opennlp/embeddings/HuggingFaceModelCacheTest*.class + + + + + + jmh diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java index 327729765a..18f0abc09a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java @@ -23,30 +23,61 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.util.ArrayList; +import java.util.HexFormat; import java.util.List; +import java.util.Optional; +import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Fetches the files a distillation needs from a Hugging Face model repository into a local cache * directory, so a teacher can be named by its hub id ({@code org/model}) instead of a local path. - * Files download once and are reused afterwards; a file the repository does not have (a 404, e.g. - * a WordPiece teacher's {@code sentencepiece.bpe.model}) is reported as absent, not an error. + * + *

A download is pinned and verified. The teacher's ref is resolved to a commit sha once, every + * file is then requested at that sha, and every file is checked against the digest the hub + * publishes for it before it is published into the cache directory. A file the hub does not + * publish a digest for is refused rather than used, the same posture + * {@link opennlp.tools.util.DownloadUtil} takes for a model whose {@code .sha512} sidecar cannot + * be read. A file the repository does not have (a 404, e.g. a WordPiece teacher's + * {@code sentencepiece.bpe.model}) is reported as absent, not an error, but only when that file is + * optional.

+ * + *

The resolved commit sha is recorded in {@value #REVISION_FILE} in the cache directory, which + * also marks the directory as a complete snapshot of that one revision: while it is there, the + * files are reused without a request to the hub and without being digested again. The record is + * written only once every file has been verified, and re-reading a multi-gigabyte ONNX graph on + * every distillation would only guard against something that could rewrite the record just as + * easily. A directory without the record is not trusted: its files are checked against the + * revision now being downloaded before they are kept. A record that does not describe the + * directory, because a file it vouches for is gone or because it names a different commit than the + * reference asks for, is removed before anything is downloaded, so that a download failing halfway + * cannot leave a directory the next attempt would trust on the strength of it.

*/ final class HuggingFaceModelCache { - /** The hub's host, the prefix of every download URL. */ + /** The hub's base URL, the prefix of every download URL. */ private static final String HUB_BASE = "https://huggingface.co/"; - /** The hub's download path between the model id and the repository-relative file name. */ - private static final String RESOLVE_PATH = "/resolve/main/"; + /** The hub's download path between the model id and the revision. */ + private static final String RESOLVE_PATH = "/resolve/"; + + /** The revision downloaded when a teacher does not name one: the repository's default branch. */ + private static final String DEFAULT_REVISION = "main"; - /** A hub model id: an organization and a model name, both of word characters, dots, or dashes. */ - private static final Pattern MODEL_ID_PATTERN = Pattern.compile("[\\w.-]+/[\\w.-]+"); + /** + * A teacher reference: an organization and a model name, both of word characters, dots, or + * dashes, optionally followed by {@code @} and the revision to pin. + */ + private static final Pattern TEACHER_PATTERN = Pattern.compile("([\\w.-]+/[\\w.-]+)(?:@([\\w.-]+))?"); /** The directory the cache lives in, below the user's home directory. */ private static final String CACHE_DIRECTORY = ".cache"; @@ -54,12 +85,46 @@ final class HuggingFaceModelCache { /** The cache's own directory, below {@link #CACHE_DIRECTORY}. */ private static final String CACHE_NAME = "opennlp-embeddings"; + /** The hex length of the digest suffix that makes a cache directory name injective. */ + private static final int CACHE_KEY_HEX_LENGTH = 16; + + /** + * The file recording the commit sha the cache directory holds. It is written only after every + * file of that revision has been downloaded and verified, so its presence means the directory is + * complete. The name starts with a dot so that it cannot collide with a repository file. + */ + static final String REVISION_FILE = ".opennlp-revision"; + /** The suffix of the temporary file a download streams into before it is moved into place. */ private static final String DOWNLOAD_SUFFIX = ".download"; - /** The HTTP status a served file answers with; anything else means the file is not there. */ + /** The response header holding the commit sha a ref resolved to. */ + private static final String COMMIT_HEADER = "x-repo-commit"; + + /** The response header holding the digest of the file, quoted. */ + private static final String ETAG_HEADER = "x-linked-etag"; + + /** The length in hex of a SHA-1: the shape of a commit sha and of a git object name. */ + private static final int SHA1_HEX_LENGTH = 40; + + /** The length in hex of a SHA-256: the shape of the digest published for a Git LFS file. */ + private static final int SHA256_HEX_LENGTH = 64; + + /** A hex string of any length, the shape both the commit sha and the digests have. */ + private static final Pattern HEX_PATTERN = Pattern.compile("[0-9a-fA-F]+"); + + /** The header git hashes in front of a blob's bytes, completed by the length and a NUL byte. */ + private static final String GIT_BLOB_PREFIX = "blob "; + + /** The read size when digesting a downloaded file. */ + private static final int DIGEST_BUFFER_SIZE = 8192; + + /** The HTTP status a served file answers with. */ private static final int HTTP_OK = 200; + /** The HTTP status of a file the repository does not have at the requested revision. */ + private static final int HTTP_NOT_FOUND = 404; + /** How long the client waits for a connection to the hub. */ private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(30); @@ -91,31 +156,78 @@ private static List optionalFiles() { } /** - * Resolves a teacher reference to a local directory holding its files. + * Resolves a teacher reference to a local directory holding its files, downloading them from the + * Hugging Face hub when the reference is a model id. * - * @param teacher A local directory, used as-is, or a Hugging Face model id - * ({@code org/model}), downloaded into + * @param teacher A local directory, used as-is, or a Hugging Face model id ({@code org/model}, + * or {@code org/model@revision} to pin a branch, tag, or commit sha instead of + * the default branch), downloaded into * {@code ~/.cache/opennlp-embeddings/org-model} on first use (the slash becomes - * a dash and dots become underscores). Must not be {@code null}. + * a dash, dots and the revision separator become underscores). Must not be + * {@code null}. * @param listener Receives one progress line per download; may be {@code null}. * @return The local teacher directory. - * @throws IllegalArgumentException Thrown if {@code teacher} is {@code null}, a local path - * that is not a directory, or a hub id whose required files cannot be downloaded. + * @throws IllegalArgumentException Thrown if {@code teacher} is {@code null}, or is neither a + * directory nor a well-formed model id. + * @throws IOException Thrown if a required file cannot be downloaded, or if a downloaded file + * cannot be verified against the digest the hub publishes for it. */ - static Path resolve(String teacher, ModelDistiller.ProgressListener listener) { + static Path resolve(String teacher, ModelDistiller.ProgressListener listener) throws IOException { + return resolve(teacher, HUB_BASE, defaultCacheRoot(), listener); + } + + /** + * Resolves a teacher reference against a given hub and cache location, the form used by the + * tests and by an installation that mirrors the hub. + * + * @param teacher The teacher reference, as in {@link #resolve(String, + * ModelDistiller.ProgressListener)}. Must not be {@code null}. + * @param hubBase The hub's base URL, ending in a slash. Must not be {@code null}. + * @param cacheRoot The directory the per-teacher cache directories live in. Must not be + * {@code null}. + * @param listener Receives one progress line per download; may be {@code null}. + * @return The local teacher directory. + * @throws IllegalArgumentException Thrown if {@code teacher}, {@code hubBase}, or + * {@code cacheRoot} is {@code null}, or if {@code teacher} is neither a directory nor a + * well-formed model id. + * @throws IOException Thrown if a required file cannot be downloaded, or if a downloaded file + * cannot be verified against the digest the hub publishes for it. + */ + static Path resolve(String teacher, String hubBase, Path cacheRoot, + ModelDistiller.ProgressListener listener) throws IOException { if (teacher == null) { throw new IllegalArgumentException("Teacher must not be null"); } + if (hubBase == null) { + throw new IllegalArgumentException("HubBase must not be null"); + } + if (cacheRoot == null) { + throw new IllegalArgumentException("CacheRoot must not be null"); + } final Path local = Path.of(teacher); if (Files.isDirectory(local)) { return local; } - if (!MODEL_ID_PATTERN.matcher(teacher).matches()) { + final Matcher reference = TEACHER_PATTERN.matcher(teacher); + if (!reference.matches()) { throw new IllegalArgumentException("Teacher '" + teacher + "' is neither a local " - + "directory nor a Hugging Face model id (expected 'org/model')"); + + "directory nor a Hugging Face model id (expected 'org/model' or 'org/model@revision')"); } - final Path cache = Path.of(System.getProperty("user.home"), CACHE_DIRECTORY, CACHE_NAME, - teacher.replace('/', '-').replace('.', '_')); + final String modelId = reference.group(1); + final String requestedRevision = reference.group(2); + final Path cache = cacheRoot.resolve(cacheDirectoryName(teacher)); + final String pinned = pinnedRevision(cache); + if (pinned != null && hasRequiredFiles(cache) + && (!isCommitSha(requestedRevision) || pinned.equalsIgnoreCase(requestedRevision))) { + return cache; + } + // The directory is not a complete snapshot of a revision this reference names, so the record + // it carries does not describe it either. The record goes before the first file is fetched: + // a download that then fails verification leaves a directory the next attempt distrusts and + // verifies file by file, rather than one it would hand out whole on the strength of a record + // written for an earlier revision. + Files.deleteIfExists(cache.resolve(REVISION_FILE)); + final String ref = requestedRevision == null ? DEFAULT_REVISION : requestedRevision; // A client built through the builder has no proxy selector unless one is set, so the // http.proxyHost / https.proxyHost system properties would otherwise be ignored. final HttpClient client = HttpClient.newBuilder() @@ -123,76 +235,281 @@ static Path resolve(String teacher, ModelDistiller.ProgressListener listener) { .proxy(ProxySelector.getDefault()) .connectTimeout(CONNECT_TIMEOUT) .build(); + final String commit = resolveCommit(client, hubBase, modelId, ref, requestedRevision); + report(listener, "Teacher " + modelId + " at " + ref + " is commit " + commit); for (final String file : REQUIRED_FILES) { - download(client, teacher, file, cache, true, listener); + download(client, hubBase, modelId, commit, file, cache, true, listener); } for (final String file : OPTIONAL_FILES) { - download(client, teacher, file, cache, false, listener); + download(client, hubBase, modelId, commit, file, cache, false, listener); } + Files.writeString(cache.resolve(REVISION_FILE), commit + System.lineSeparator(), + StandardCharsets.UTF_8); return cache; } /** - * Downloads one repository file into the cache, skipping files already there. + * {@return the commit sha a cache directory was downloaded at, or {@code null} when the + * directory is not a complete cached snapshot of a hub revision} + * + * @param teacherDirectory The directory to read; need not exist. + */ + static String pinnedRevision(Path teacherDirectory) { + final Path file = teacherDirectory.resolve(REVISION_FILE); + if (!Files.isRegularFile(file)) { + return null; + } + try { + final String recorded = Files.readString(file, StandardCharsets.UTF_8).trim(); + return isCommitSha(recorded) ? recorded : null; + } catch (IOException e) { + return null; + } + } + + /** {@return the directory the per-teacher cache directories live in} */ + private static Path defaultCacheRoot() { + return Path.of(System.getProperty("user.home"), CACHE_DIRECTORY, CACHE_NAME); + } + + /** + * {@return the cache directory name for a teacher reference} + * + *

The readable part replaces the characters a path cannot carry, which alone is not + * injective: {@code acme/model@v1}, {@code acme/model.v1} and {@code acme/model_v1} would all + * name one directory, and the cached fast path answers from that directory without contacting + * the hub, so one teacher would be served another's files. The suffix is a digest of the exact + * reference, so distinct references never share a directory.

+ * + * @param teacher The teacher reference, as the caller wrote it. + */ + static String cacheDirectoryName(String teacher) { + final String readable = teacher.replace('/', '-').replace('.', '_').replace('@', '_'); + final MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is required of every JVM", e); + } + final byte[] hash = digest.digest(teacher.getBytes(StandardCharsets.UTF_8)); + final String suffix = HexFormat.of().formatHex(hash, 0, CACHE_KEY_HEX_LENGTH / 2); + return readable + '-' + suffix; + } + + /** + * {@return whether every file a distillation needs is in the cache directory} + * + * @param cache The cache directory. + */ + private static boolean hasRequiredFiles(Path cache) { + for (final String file : REQUIRED_FILES) { + if (!Files.isRegularFile(cache.resolve(file))) { + return false; + } + } + return true; + } + + /** + * Resolves a ref to the commit sha it points at, so that the files of one download all come from + * one revision even if the ref moves while the download runs. The hub reports the sha on every + * resolve response, so the body of the probed file is not read. + * + * @param client The HTTP client. + * @param hubBase The hub's base URL. + * @param modelId The hub model id. + * @param ref The revision to resolve. + * @param requestedRevision The revision the teacher reference named, or {@code null} when it + * named none. + * @return The commit sha, 40 hex characters. + * @throws IOException Thrown if the ref cannot be resolved. + */ + private static String resolveCommit(HttpClient client, String hubBase, String modelId, String ref, + String requestedRevision) throws IOException { + final String probe = REQUIRED_FILES.get(0); + final HttpResponse response = send(client, hubBase, modelId, ref, probe); + // The headers carry everything this probe wants, so the body is closed unread. + final InputStream body = response.body(); + try (body) { + if (response.statusCode() != HTTP_OK) { + throw new IOException("Failed to resolve revision '" + ref + "' of " + modelId + ": HTTP " + + response.statusCode() + " for " + probe); + } + final String commit = originHeader(response, COMMIT_HEADER); + if (!isCommitSha(commit)) { + throw new IOException("Revision '" + ref + "' of " + modelId + " could not be pinned: the " + + "hub sent " + (commit == null ? "no " + COMMIT_HEADER + " header" + : COMMIT_HEADER + " '" + commit + "', which is not a commit sha") + + "; refusing to download files that cannot be attributed to one revision"); + } + if (isCommitSha(requestedRevision) && !commit.equalsIgnoreCase(requestedRevision)) { + throw new IOException("Revision '" + requestedRevision + "' of " + modelId + " resolved to " + + "commit " + commit + " instead"); + } + return commit; + } + } + + /** + * Downloads one repository file at a pinned revision into the cache, keeping a copy that is + * already there when it matches the revision's digest. * * @param client The HTTP client. + * @param hubBase The hub's base URL. * @param modelId The hub model id. + * @param commit The commit sha every file of this download is requested at. * @param file The repository-relative file name. * @param cache The cache directory. - * @param required Whether a missing file is an error. + * @param required Whether a file the revision does not have is an error. * @param listener The progress listener; may be {@code null}. - * @throws IllegalArgumentException Thrown if a required file cannot be downloaded. + * @throws IOException Thrown if a required file cannot be downloaded, or if the download cannot + * be verified against the digest the hub publishes for it. */ - private static void download(HttpClient client, String modelId, String file, Path cache, - boolean required, ModelDistiller.ProgressListener listener) { + private static void download(HttpClient client, String hubBase, String modelId, String commit, + String file, Path cache, boolean required, + ModelDistiller.ProgressListener listener) throws IOException { final Path target = cache.resolve(file); - if (Files.isRegularFile(target)) { - return; - } - final HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(HUB_BASE + modelId + RESOLVE_PATH + file)) - .timeout(DOWNLOAD_TIMEOUT) - .GET() - .build(); - final HttpResponse response; - try { - response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to download " + file + " of " + modelId + ": " - + e.getMessage(), e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalArgumentException("Interrupted while downloading " + file + " of " - + modelId, e); - } + final HttpResponse response = send(client, hubBase, modelId, commit, file); Path temporary = null; try (InputStream body = response.body()) { - if (response.statusCode() != HTTP_OK) { - if (required) { - throw new IllegalArgumentException("Failed to download " + file + " of " + modelId - + ": HTTP " + response.statusCode() + "; the distillation needs this file"); - } + if (response.statusCode() == HTTP_NOT_FOUND && !required) { + // The cache directory holds one revision: a copy left by an earlier one has to go. + Files.deleteIfExists(target); return; } - if (listener != null) { - listener.progress("Downloading " + modelId + "/" + file + " ..."); + if (response.statusCode() != HTTP_OK) { + throw new IOException("Failed to download " + file + " of " + modelId + " at commit " + + commit + ": HTTP " + response.statusCode() + + (required ? "; the distillation needs this file" : "")); } + final Digest expected = expectedDigest(response, modelId, file); + if (Files.isRegularFile(target) && expected.matches(target)) { + return; + } + report(listener, "Downloading " + modelId + "/" + file + " ..."); Files.createDirectories(target.getParent()); // A temporary name unique per download: two processes sharing one cache directory must not // stream two copies of the same file into one partial file and publish the interleaving. temporary = Files.createTempFile(target.getParent(), target.getFileName().toString(), DOWNLOAD_SUFFIX); Files.copy(body, temporary, StandardCopyOption.REPLACE_EXISTING); + final String actual = expected.form().hexOf(temporary); + if (!expected.hex().equalsIgnoreCase(actual)) { + throw new IOException(expected.form().displayName() + " checksum validation failed for " + + file + " of " + modelId + " at commit " + commit + ". Expected: " + expected.hex() + + ", but got: " + actual); + } Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); temporary = null; - } catch (IOException e) { - throw new IllegalArgumentException("Failed to store " + file + " of " + modelId + " at " - + target + ": " + e.getMessage(), e); } finally { deleteIfPresent(temporary); } } + /** + * Sends one GET to the hub. + * + * @param client The HTTP client. + * @param hubBase The hub's base URL. + * @param modelId The hub model id. + * @param revision The revision to request the file at. + * @param file The repository-relative file name. + * @return The response, whose body has not been read yet. + * @throws IOException Thrown if the request fails. + */ + private static HttpResponse send(HttpClient client, String hubBase, String modelId, + String revision, String file) throws IOException { + final HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(hubBase + modelId + RESOLVE_PATH + revision + "/" + file)) + .timeout(DOWNLOAD_TIMEOUT) + .GET() + .build(); + try { + return client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + } catch (IOException e) { + throw new IOException("Failed to download " + file + " of " + modelId + ": " + + e.getMessage(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while downloading " + file + " of " + modelId, e); + } + } + + /** + * Reads the digest the hub publishes for a file, refusing a file that carries none. + * + * @param response The response. + * @param modelId The hub model id, for the message. + * @param file The repository-relative file name, for the message. + * @return The expected digest, either a git blob SHA-1 or a SHA-256. + * @throws IOException Thrown if the header is absent or is not one of the two digest forms. + */ + private static Digest expectedDigest(HttpResponse response, String modelId, + String file) throws IOException { + final String header = originHeader(response, ETAG_HEADER); + if (header == null) { + throw new IOException("Expected checksum could not be retrieved for " + file + " of " + + modelId + ": the hub sent no " + ETAG_HEADER + " header; refusing to use a file that " + + "cannot be verified"); + } + final String hex = header.trim().replace("\"", ""); + final Checksum form = Checksum.of(hex); + if (form == null) { + throw new IOException("Expected checksum could not be retrieved for " + file + " of " + + modelId + ": " + ETAG_HEADER + " '" + header + "' is neither a git blob SHA-1 nor a " + + "SHA-256; refusing to use a file that cannot be verified"); + } + return new Digest(form, hex); + } + + /** + * {@return the value the hub sent for a header, or {@code null} when it sent none} + * + *

A resolve request answers with a redirect to a content delivery network, and the client + * does not copy the headers of that redirecting response onto the response it finally returns, + * so the redirect chain is walked back to its start. The earliest response wins: the digest to + * verify against is the one the hub itself stated, not one a later hop restated.

+ * + * @param response The response, at the end of its redirect chain. + * @param name The header name. + */ + private static String originHeader(HttpResponse response, String name) { + final List> chain = new ArrayList<>(); + for (HttpResponse hop = response; hop != null; + hop = hop.previousResponse().orElse(null)) { + chain.add(hop); + } + for (int i = chain.size() - 1; i >= 0; i--) { + final Optional value = chain.get(i).headers().firstValue(name); + if (value.isPresent()) { + return value.get(); + } + } + return null; + } + + /** + * {@return whether a value is a commit sha, 40 hex characters} + * + * @param value The value to check; may be {@code null}. + */ + private static boolean isCommitSha(String value) { + return value != null && value.length() == SHA1_HEX_LENGTH + && HEX_PATTERN.matcher(value).matches(); + } + + /** + * Reports one progress line, if anyone is listening. + * + * @param listener The listener; may be {@code null}. + * @param message The message. + */ + private static void report(ModelDistiller.ProgressListener listener, String message) { + if (listener != null) { + listener.progress(message); + } + } + /** * Deletes a partial download, if there is one, without reporting a failure to do so. * @@ -208,4 +525,93 @@ private static void deleteIfPresent(Path file) { // A leftover partial download costs disk space; the next attempt writes a fresh file. } } + + /** + * The digest the hub published for one file. + * + * @param form The digest form the hub stated it in. + * @param hex The digest value in hex, without the quotes the header carries. + */ + private record Digest(Checksum form, String hex) { + + /** + * {@return whether a file digests to the value the hub published} + * + * @param file The file to digest. + * @throws IOException Thrown if the file cannot be read. + */ + boolean matches(Path file) throws IOException { + return hex.equalsIgnoreCase(form.hexOf(file)); + } + } + + /** + * The two digest forms the hub publishes in its {@code x-linked-etag} header, told apart by the + * length of the hex value. + */ + private enum Checksum { + + /** The git object name of a file stored in git itself: its bytes behind a blob header. */ + GIT_BLOB_SHA1("git blob SHA-1", "SHA-1", SHA1_HEX_LENGTH), + + /** The digest of a file stored in Git LFS: its bytes alone. */ + LFS_SHA256("SHA-256", "SHA-256", SHA256_HEX_LENGTH); + + private final String displayName; + private final String algorithm; + private final int hexLength; + + Checksum(String displayName, String algorithm, int hexLength) { + this.displayName = displayName; + this.algorithm = algorithm; + this.hexLength = hexLength; + } + + /** + * {@return the digest form a hex value of this length is, or {@code null} when the value is + * not a hex string of either length} + * + * @param value The digest value, without its quotes. Must not be {@code null}. + */ + static Checksum of(String value) { + for (final Checksum checksum : values()) { + if (value.length() == checksum.hexLength && HEX_PATTERN.matcher(value).matches()) { + return checksum; + } + } + return null; + } + + /** {@return the name of this digest form, for a message} */ + String displayName() { + return displayName; + } + + /** + * {@return the digest of a file in this form, in lower case hex} + * + * @param file The file to digest. + * @throws IOException Thrown if the file cannot be read. + */ + String hexOf(Path file) throws IOException { + final MessageDigest digest; + try { + digest = MessageDigest.getInstance(algorithm); + } catch (NoSuchAlgorithmException e) { + throw new IOException(algorithm + " is not available", e); + } + if (this == GIT_BLOB_SHA1) { + digest.update((GIT_BLOB_PREFIX + Files.size(file) + '\0') + .getBytes(StandardCharsets.US_ASCII)); + } + try (InputStream in = Files.newInputStream(file); + DigestInputStream digesting = new DigestInputStream(in, digest)) { + final byte[] buffer = new byte[DIGEST_BUFFER_SIZE]; + while (digesting.read(buffer) != -1) { + // Reading the file is what updates the digest. + } + } + return HexFormat.of().formatHex(digest.digest()); + } + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java index 369cd382c3..a602cbe43a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java @@ -90,8 +90,8 @@ public record Result(String family, int vocabularySize, int teacherDimension, in /** * Distills a teacher into a model directory, resolving the teacher reference first: a local - * directory is used as-is, a Hugging Face model id ({@code org/model}) is downloaded into a - * local cache on first use. + * directory is used as-is, a Hugging Face model id ({@code org/model}, or + * {@code org/model@revision} to pin a revision) is downloaded into a local cache on first use. * * @param teacher The teacher: a local directory or a Hugging Face model id. Must not * be {@code null}. @@ -100,8 +100,9 @@ public record Result(String family, int vocabularySize, int teacherDimension, in * @param listener Receives progress lines; may be {@code null}. * @return The distillation result, read back from the verified directory. * @throws IllegalArgumentException Thrown if an argument is {@code null} or invalid, the - * teacher cannot be resolved, or the teacher cannot be run. - * @throws IOException Thrown if reading or writing a file fails. + * teacher reference is malformed, or the teacher cannot be run. + * @throws IOException Thrown if reading or writing a file fails, or if a teacher cannot be + * downloaded and verified. */ public static Result distill(String teacher, Path outputDirectory, int pcaDims, ProgressListener listener) throws IOException { @@ -307,6 +308,7 @@ private static String configJson(Path teacherDirectory, int pcaDims, int compone + " \"model_type\": \"model2vec\",\n" + " \"architectures\": [\"StaticModel\"],\n" + " \"tokenizer_name\": \"" + (name == null ? teacherDirectory : name) + "\",\n" + + teacherRevisionField(teacherDirectory) + " \"apply_pca\": " + pcaDims + ",\n" + " \"sif_coefficient\": " + SIF_COEFFICIENT + ",\n" + " \"hidden_dim\": " + components + ",\n" @@ -317,6 +319,21 @@ private static String configJson(Path teacherDirectory, int pcaDims, int compone + "}\n"; } + /** + * {@return the {@code config.json} field naming the commit the teacher's files came from, or an + * empty string when the teacher directory is not a cached hub download} + * + *

A distilled table is not reproducible without the exact revision of the teacher it was + * distilled from, and the teacher can move under its branch name, so the sha travels with the + * table rather than only staying in the cache directory it was downloaded into.

+ * + * @param teacherDirectory The teacher's directory. + */ + private static String teacherRevisionField(Path teacherDirectory) { + final String revision = HuggingFaceModelCache.pinnedRevision(teacherDirectory); + return revision == null ? "" : " \"teacher_revision\": \"" + revision + "\",\n"; + } + /** * Copies the teacher's trained SentencePiece {@code .model} file into the model directory when * the teacher has one; the distillation cannot fabricate it and the loader needs it for the diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java index 7f3f662a3a..56d74f545d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java @@ -28,8 +28,9 @@ interface DistillModelParams { * {@return the teacher to distill: a local directory or a Hugging Face model id} */ @ParameterDescription(valueName = "hf-id-or-path", - description = "The sentence-transformer teacher: a Hugging Face model id (org/model) or a " - + "local directory holding tokenizer.json and onnx/model.onnx.") + description = "The sentence-transformer teacher: a Hugging Face model id (org/model, or " + + "org/model@revision to pin a branch, tag, or commit) or a local directory holding " + + "tokenizer.json and onnx/model.onnx.") String getTeacher(); /** diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java index 8ea19f376d..5d0c7a74ff 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java @@ -29,8 +29,9 @@ * (teacher forward pass over the vocabulary, PCA, Zipf weighting) in Java, so producing a table * no longer needs a Python environment; see {@link ModelDistiller} for the pipeline. * - *

The teacher is a Hugging Face model id (its files download once into a local cache) or a - * local directory holding {@code tokenizer.json} and {@code onnx/model.onnx}. A SentencePiece + *

The teacher is a Hugging Face model id (its files download once into a local cache, pinned to + * the commit its revision resolved to and verified against the digests the hub publishes for them) + * or a local directory holding {@code tokenizer.json} and {@code onnx/model.onnx}. A SentencePiece * teacher also needs its trained {@code .model} file, downloaded or supplied alongside. The * written directory is completed and verified by loading it, so a run that prints a summary is a * directory that works.

diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java index 52180b837c..d97e13e450 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HuggingFaceModelCacheTest.java @@ -17,25 +17,142 @@ package opennlp.embeddings; import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * The teacher-reference contract of the cache, exercised without touching the network: a local - * directory is returned as-is and anything that is neither a directory nor an {@code org/model} - * hub id is rejected before a request is made. + * The cache's teacher-reference contract and its download integrity, exercised against a hub + * served on the loopback interface: no test here reaches the network. A local directory is + * returned as-is, anything that is neither a directory nor an {@code org/model} hub id is rejected + * before a request is made, and a download is pinned to one commit and refused unless it matches + * the digest the hub published for it. */ class HuggingFaceModelCacheTest { + /** The address the test hub binds to, so that a test cannot leave the machine. */ + private static final String LOOPBACK = "127.0.0.1"; + + /** The model id of the teacher the hub serves. */ + private static final String MODEL_ID = "acme/teacher"; + + /** The cache directory name {@link #MODEL_ID} maps to, derived rather than restated. */ + private static final String CACHE_NAME = HuggingFaceModelCache.cacheDirectoryName(MODEL_ID); + + /** + * {@return the cache directory name for {@link #MODEL_ID} pinned to a revision} + * + * @param revision The revision the reference names. + */ + private static String cacheNameAt(String revision) { + return HuggingFaceModelCache.cacheDirectoryName(MODEL_ID + "@" + revision); + } + + /** The ref a teacher reference without a revision resolves. */ + private static final String DEFAULT_REF = "main"; + + /** The commit {@link #DEFAULT_REF} resolves to, a sha of the shape the hub reports. */ + private static final String COMMIT = "1110a243fdf4706b3f48f1d95db1a4f5529b4d41"; + + /** A second commit, for the teacher that moved under its ref. */ + private static final String OTHER_COMMIT = "0f2b8b1d4c7e6a5938271605f4e3d2c1b0a99887"; + + /** The tokenizer the hub serves, a file small enough for git to store it as a blob. */ + private static final byte[] TOKENIZER = bytes("{\"model\":{\"type\":\"WordPiece\"}}\n"); + + /** + * The git blob SHA-1 of {@link #TOKENIZER}, the 40 character form of the etag: this value comes + * from {@code git hash-object} over the same bytes, not from the code under test. + */ + private static final String TOKENIZER_BLOB_SHA1 = "296101682cfaaf7c2d1e2394062858aea9dd3ea5"; + + /** + * The SHA-1 of {@link #TOKENIZER}'s bytes alone, which is not how git names a blob: git hashes + * the length and a NUL byte in front of the content. + */ + private static final String TOKENIZER_PLAIN_SHA1 = "4d02516eda32c9ae5c590766d9e055835e0bb2c7"; + + /** The ONNX graph the hub serves, large enough in reality to be stored in Git LFS. */ + private static final byte[] ONNX = bytes("ONNX GRAPH BYTES\n"); + + /** The SHA-256 of {@link #ONNX}, the 64 character form of the etag, from {@code sha256sum}. */ + private static final String ONNX_SHA256 = + "faffaa0a29c6cf303b7a0dfc59d54131b17b2658c22e02c5da3a66d7526360ef"; + + /** + * A tokenizer larger than the buffer a download is digested in, so that a digest taken from a + * single read instead of a loop over the whole file would not match. + */ + private static final byte[] BIG_TOKENIZER = repeated('x', 20000); + + /** The git blob SHA-1 of {@link #BIG_TOKENIZER}, from {@code git hash-object}. */ + private static final String BIG_TOKENIZER_BLOB_SHA1 = + "7eded2aa2b98c9f0d9d4bb82c277cbbd09dcd044"; + + /** An ONNX graph larger than that buffer, for the SHA-256 form. */ + private static final byte[] BIG_ONNX = repeated('y', 20000); + + /** The SHA-256 of {@link #BIG_ONNX}, from {@code sha256sum}. */ + private static final String BIG_ONNX_SHA256 = + "fdb7f88419c3dd0053ff7c3e9db63fda5bcedf3b8a7344fc1a955a17f4423b58"; + + /** The tokenizer configuration the hub serves, an optional file. */ + private static final byte[] TOKENIZER_CONFIG = bytes("{\"do_lower_case\":true}\n"); + + /** The git blob SHA-1 of {@link #TOKENIZER_CONFIG}, from {@code git hash-object}. */ + private static final String TOKENIZER_CONFIG_BLOB_SHA1 = + "67a56d358bc09865322d344d13922261a6277f26"; + + /** The SentencePiece model the hub serves, an optional file. */ + private static final byte[] SENTENCEPIECE = bytes("SPM\n"); + + /** The git blob SHA-1 of {@link #SENTENCEPIECE}, from {@code git hash-object}. */ + private static final String SENTENCEPIECE_BLOB_SHA1 = + "91a9c1344fe72a78cc937f3cc515050ab1b52f20"; + + /** The first of the SentencePiece file names the cache tries. */ + private static final String SENTENCEPIECE_MODEL = ModelFileNames.SENTENCEPIECE_MODELS.get(0); + + /** The HTTP status of a file a revision does not have. */ + private static final int NOT_FOUND = 404; + + private Hub hub; + + @BeforeEach + void startHub() throws IOException { + hub = new Hub(); + } + + @AfterEach + void stopHub() { + hub.close(); + } + @Test void testNullTeacherFailsLoudly() { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, @@ -44,17 +161,31 @@ void testNullTeacherFailsLoudly() { } @Test - void testLocalDirectoryIsUsedAsIs(@TempDir Path teacher) { + void testNullHubBaseFailsLoudly(@TempDir Path cacheRoot) { + assertEquals("HubBase must not be null", assertThrows(IllegalArgumentException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, null, cacheRoot, null)).getMessage()); + } + + @Test + void testNullCacheRootFailsLoudly() { + assertEquals("CacheRoot must not be null", assertThrows(IllegalArgumentException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), null, null)).getMessage()); + } + + @Test + void testLocalDirectoryIsUsedAsIs(@TempDir Path teacher) throws IOException { assertEquals(teacher, HuggingFaceModelCache.resolve(teacher.toString(), null)); } @ParameterizedTest @ValueSource(strings = {"bge-m3", "BAAI/bge m3", "BAAI/bge-m3/onnx", "/BAAI/bge-m3", - "BAAI/bge-m3/", "BAAI//bge-m3"}) - void testMalformedTeacherReferenceIsRejectedBeforeAnyRequest(String teacher) { + "BAAI/bge-m3/", "BAAI//bge-m3", "BAAI/bge-m3@", "BAAI/bge-m3@a b", "BAAI/bge-m3@main@main"}) + void testMalformedTeacherReferenceIsRejectedBeforeAnyRequest(String teacher, + @TempDir Path cacheRoot) { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> HuggingFaceModelCache.resolve(teacher, null)); + () -> HuggingFaceModelCache.resolve(teacher, hub.base(), cacheRoot, null)); assertTrue(e.getMessage().contains("org/model"), e.getMessage()); + assertTrue(hub.requests.isEmpty(), hub.requests.toString()); } /** @@ -77,4 +208,628 @@ void testAnExistingRegularFileIsRejected(@TempDir Path root) throws IOException () -> HuggingFaceModelCache.resolve(file.toString(), null)); assertTrue(e.getMessage().contains("org/model"), e.getMessage()); } + + /** + * The two digest forms the hub uses, on the two files a distillation needs: a git blob SHA-1 for + * a file stored in git and a SHA-256 for one stored in Git LFS. + */ + @Test + void testDownloadsAndVerifiesBothEtagForms(@TempDir Path cacheRoot) throws IOException { + serveTeacher(); + final List progress = new ArrayList<>(); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, + progress::add); + + assertEquals(cacheRoot.resolve(CACHE_NAME), cache); + assertArrayEquals(TOKENIZER, Files.readAllBytes(cache.resolve(ModelFileNames.TOKENIZER_JSON))); + assertArrayEquals(ONNX, Files.readAllBytes(cache.resolve(ModelFileNames.ONNX_MODEL))); + assertEquals(COMMIT, HuggingFaceModelCache.pinnedRevision(cache)); + assertTrue(progress.stream().anyMatch(line -> line.contains(COMMIT)), progress.toString()); + } + + /** The recorded revision is what a reader of the cache directory finds, in plain text. */ + @Test + void testTheResolvedCommitIsRecordedInTheCacheDirectory(@TempDir Path cacheRoot) + throws IOException { + serveTeacher(); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertEquals(COMMIT, + Files.readString(cache.resolve(HuggingFaceModelCache.REVISION_FILE)).trim()); + } + + /** + * The ref is resolved once and every file is then asked for by commit sha, so that a ref moving + * mid-download cannot mix two revisions into one cache directory. + */ + @Test + void testEveryFileIsRequestedAtTheResolvedCommit(@TempDir Path cacheRoot) throws IOException { + serveTeacher(); + + HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertEquals(1, hub.requests.stream().filter(p -> p.contains("/" + DEFAULT_REF + "/")).count(), + hub.requests.toString()); + assertTrue(hub.requests.stream().filter(p -> !p.contains("/" + DEFAULT_REF + "/")) + .allMatch(p -> p.startsWith("/" + MODEL_ID + "/resolve/" + COMMIT + "/")), + hub.requests.toString()); + } + + @Test + void testACorruptedBodyIsRejected(@TempDir Path cacheRoot) throws IOException { + serveTeacher(); + hub.serve(COMMIT, ModelFileNames.ONNX_MODEL, bytes("not the graph the hub promised\n"), + quoted(ONNX_SHA256)); + + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains("SHA-256 checksum validation failed"), e.getMessage()); + assertTrue(e.getMessage().contains(ModelFileNames.ONNX_MODEL), e.getMessage()); + assertTrue(e.getMessage().contains(ONNX_SHA256), e.getMessage()); + assertTrue(e.getMessage().contains("but got:"), e.getMessage()); + assertNothingUsable(cacheRoot.resolve(CACHE_NAME), ModelFileNames.ONNX_MODEL); + } + + /** + * The 40 character etag is the git blob SHA-1, not the SHA-1 of the content, and a file that + * only matches the latter is a file whose length git would disagree about. + */ + @Test + void testThePlainSha1OfTheContentIsNotAcceptedAsTheGitBlobSha1(@TempDir Path cacheRoot) { + serveTeacher(); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_PLAIN_SHA1)); + + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains("git blob SHA-1 checksum validation failed"), + e.getMessage()); + assertTrue(e.getMessage().contains(TOKENIZER_BLOB_SHA1), e.getMessage()); + } + + /** + * A download is digested by reading it in a loop, so a file longer than one of those reads is + * digested whole, in both of the forms the hub publishes. The expected values come from + * {@code git hash-object} and {@code sha256sum} over the same bytes. + */ + @Test + void testABodyLongerThanTheDigestBufferIsDigestedWhole(@TempDir Path cacheRoot) + throws IOException { + hub.serve(DEFAULT_REF, ModelFileNames.TOKENIZER_JSON, BIG_TOKENIZER, + quoted(BIG_TOKENIZER_BLOB_SHA1)); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, BIG_TOKENIZER, + quoted(BIG_TOKENIZER_BLOB_SHA1)); + hub.serve(COMMIT, ModelFileNames.ONNX_MODEL, BIG_ONNX, quoted(BIG_ONNX_SHA256)); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertArrayEquals(BIG_TOKENIZER, + Files.readAllBytes(cache.resolve(ModelFileNames.TOKENIZER_JSON))); + assertArrayEquals(BIG_ONNX, Files.readAllBytes(cache.resolve(ModelFileNames.ONNX_MODEL))); + } + + /** Hex is hex: a hub that states its digests in upper case is verified against just the same. */ + @Test + void testAnEtagInUpperCaseIsAccepted(@TempDir Path cacheRoot) throws IOException { + serveTeacher(); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, + quoted(TOKENIZER_BLOB_SHA1.toUpperCase(Locale.ROOT))); + hub.serve(COMMIT, ModelFileNames.ONNX_MODEL, ONNX, + quoted(ONNX_SHA256.toUpperCase(Locale.ROOT))); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertArrayEquals(TOKENIZER, Files.readAllBytes(cache.resolve(ModelFileNames.TOKENIZER_JSON))); + assertArrayEquals(ONNX, Files.readAllBytes(cache.resolve(ModelFileNames.ONNX_MODEL))); + assertEquals(COMMIT, HuggingFaceModelCache.pinnedRevision(cache)); + } + + @Test + void testAMissingEtagIsRefused(@TempDir Path cacheRoot) throws IOException { + serveTeacher(); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, null); + + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains("Expected checksum could not be retrieved"), e.getMessage()); + assertTrue(e.getMessage().contains(ModelFileNames.TOKENIZER_JSON), e.getMessage()); + assertNothingUsable(cacheRoot.resolve(CACHE_NAME), ModelFileNames.TOKENIZER_JSON); + } + + @ParameterizedTest + @ValueSource(strings = {"", "not-a-digest", "296101682cfaaf7c2d1e2394062858aea9dd3ea", + "296101682cfaaf7c2d1e2394062858aea9dd3ea55", "zzz101682cfaaf7c2d1e2394062858aea9dd3ea5", + "sha256:faffaa0a29c6cf303b7a0dfc59d54131b17b2658c22e02c5da3a66d7526360ef"}) + void testAMalformedEtagIsRefused(String etag, @TempDir Path cacheRoot) { + serveTeacher(); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(etag)); + + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains("Expected checksum could not be retrieved"), e.getMessage()); + assertTrue(e.getMessage().contains("neither a git blob SHA-1 nor a SHA-256"), e.getMessage()); + } + + /** A file the repository does not have is absent, and one it has is downloaded and verified. */ + @Test + void testOptionalFilesAreDownloadedWhenPresentAndAbsentOnA404(@TempDir Path cacheRoot) + throws IOException { + serveTeacher(); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_CONFIG, TOKENIZER_CONFIG, + quoted(TOKENIZER_CONFIG_BLOB_SHA1)); + hub.serve(COMMIT, SENTENCEPIECE_MODEL, SENTENCEPIECE, quoted(SENTENCEPIECE_BLOB_SHA1)); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertArrayEquals(TOKENIZER_CONFIG, + Files.readAllBytes(cache.resolve(ModelFileNames.TOKENIZER_CONFIG))); + assertArrayEquals(SENTENCEPIECE, Files.readAllBytes(cache.resolve(SENTENCEPIECE_MODEL))); + // The hub was asked for the external ONNX weights and answered 404, which is not an error. + assertTrue(hub.requests.contains(resolvePath(COMMIT, ModelFileNames.ONNX_MODEL_DATA)), + hub.requests.toString()); + assertTrue(Files.notExists(cache.resolve(ModelFileNames.ONNX_MODEL_DATA))); + } + + @Test + void testAMissingRequiredFileFails(@TempDir Path cacheRoot) { + serveTeacher(); + hub.status(COMMIT, ModelFileNames.ONNX_MODEL, NOT_FOUND); + + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains(ModelFileNames.ONNX_MODEL), e.getMessage()); + assertTrue(e.getMessage().contains("the distillation needs this file"), e.getMessage()); + } + + /** Only a 404 means absent: a hub that is broken or refuses access must not look like one. */ + @Test + void testAnOptionalFileServedWithAnErrorStatusIsNotTreatedAsAbsent(@TempDir Path cacheRoot) { + serveTeacher(); + hub.status(COMMIT, ModelFileNames.TOKENIZER_CONFIG, 503); + + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains(ModelFileNames.TOKENIZER_CONFIG), e.getMessage()); + assertTrue(e.getMessage().contains("HTTP 503"), e.getMessage()); + } + + /** + * The hub answers a resolve request with a redirect to a content delivery network and states the + * commit and the digest on the redirecting response, which the client does not carry over to the + * response it finally returns. + */ + @Test + void testTheHeadersOfARedirectingResponseAreUsed(@TempDir Path cacheRoot) throws IOException { + hub.redirect(DEFAULT_REF, ModelFileNames.TOKENIZER_JSON, quoted(TOKENIZER_BLOB_SHA1), + "/cdn/tokenizer"); + hub.redirect(COMMIT, ModelFileNames.TOKENIZER_JSON, quoted(TOKENIZER_BLOB_SHA1), + "/cdn/tokenizer"); + hub.redirect(COMMIT, ModelFileNames.ONNX_MODEL, quoted(ONNX_SHA256), "/cdn/onnx"); + hub.reply("/cdn/tokenizer", new Reply(200, null, null, null, TOKENIZER)); + hub.reply("/cdn/onnx", new Reply(200, null, null, null, ONNX)); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertArrayEquals(TOKENIZER, Files.readAllBytes(cache.resolve(ModelFileNames.TOKENIZER_JSON))); + assertArrayEquals(ONNX, Files.readAllBytes(cache.resolve(ModelFileNames.ONNX_MODEL))); + assertEquals(COMMIT, HuggingFaceModelCache.pinnedRevision(cache)); + } + + /** A complete cache directory is a usable teacher with the hub unreachable. */ + @Test + void testACompleteCacheIsReusedWithoutContactingTheHub(@TempDir Path cacheRoot) + throws IOException { + serveTeacher(); + final Path first = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + hub.replies.clear(); + hub.requests.clear(); + + final Path second = HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertEquals(first, second); + assertTrue(hub.requests.isEmpty(), hub.requests.toString()); + assertArrayEquals(TOKENIZER, Files.readAllBytes(second.resolve(ModelFileNames.TOKENIZER_JSON))); + } + + /** A recorded revision without the files it vouches for is not a cache directory. */ + @Test + void testAMarkedCacheMissingItsFilesIsDownloadedAgain(@TempDir Path cacheRoot) + throws IOException { + serveTeacher(); + final Path cache = Files.createDirectories(cacheRoot.resolve(CACHE_NAME)); + Files.writeString(cache.resolve(HuggingFaceModelCache.REVISION_FILE), COMMIT); + + HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertArrayEquals(ONNX, Files.readAllBytes(cache.resolve(ModelFileNames.ONNX_MODEL))); + } + + /** + * A run that stops on a failed verification must leave nothing the next run would trust. The + * record of the revision the directory used to hold is dropped before the first file is fetched, + * so a retry checks what is on disk against the hub instead of handing out a directory half + * replaced by a revision it never finished downloading. + */ + @Test + void testAFailedVerificationLeavesNoTrustedCacheBehind(@TempDir Path cacheRoot) + throws IOException { + final Path cache = Files.createDirectories(cacheRoot.resolve(CACHE_NAME)); + Files.createDirectories(cache.resolve(ModelFileNames.ONNX_MODEL).getParent()); + // A directory marked complete whose tokenizer is gone: its graph is the earlier revision's. + Files.write(cache.resolve(ModelFileNames.ONNX_MODEL), bytes("an older revision\n")); + Files.writeString(cache.resolve(HuggingFaceModelCache.REVISION_FILE), COMMIT + "\n"); + serveTeacher(); + hub.serve(COMMIT, ModelFileNames.ONNX_MODEL, bytes("not the graph the hub promised\n"), + quoted(ONNX_SHA256)); + + assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertNull(HuggingFaceModelCache.pinnedRevision(cache)); + hub.replies.clear(); + assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null), + "the half-replaced directory must not be handed out"); + } + + /** Only a commit sha names a teacher, so a stray file cannot make a directory look pinned. */ + @Test + void testAnUnusableRevisionFileIsNotAPin(@TempDir Path cache) throws IOException { + assertNull(HuggingFaceModelCache.pinnedRevision(cache)); + + Files.writeString(cache.resolve(HuggingFaceModelCache.REVISION_FILE), "not a commit sha"); + assertNull(HuggingFaceModelCache.pinnedRevision(cache)); + + Files.writeString(cache.resolve(HuggingFaceModelCache.REVISION_FILE), COMMIT + "\n"); + assertEquals(COMMIT, HuggingFaceModelCache.pinnedRevision(cache)); + } + + /** + * A cache directory left incomplete by an interrupted run does not carry the revision it was + * downloaded at, so its files are checked against the revision now being downloaded instead of + * being trusted, and the expensive ones are not fetched again when they already match. + */ + @Test + void testAnUnmarkedCachedFileThatMatchesTheRevisionIsKept(@TempDir Path cacheRoot) + throws IOException { + serveTeacher(); + final Path cache = Files.createDirectories(cacheRoot.resolve(CACHE_NAME)); + Files.write(cache.resolve(ModelFileNames.TOKENIZER_JSON), TOKENIZER); + // A body that would fail verification: reaching it means the cached file was not reused. + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, bytes("re-downloaded\n"), + quoted(TOKENIZER_BLOB_SHA1)); + + HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertArrayEquals(TOKENIZER, Files.readAllBytes(cache.resolve(ModelFileNames.TOKENIZER_JSON))); + } + + @Test + void testAnUnmarkedCachedFileFromAnotherRevisionIsReplaced(@TempDir Path cacheRoot) + throws IOException { + serveTeacher(); + final Path cache = Files.createDirectories(cacheRoot.resolve(CACHE_NAME)); + Files.write(cache.resolve(ModelFileNames.TOKENIZER_JSON), bytes("an older revision\n")); + + HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertArrayEquals(TOKENIZER, Files.readAllBytes(cache.resolve(ModelFileNames.TOKENIZER_JSON))); + } + + /** A cache directory holds one revision, so a file the new one does not have has to go. */ + @Test + void testAnOptionalFileTheRevisionDoesNotHaveIsRemovedFromTheCache(@TempDir Path cacheRoot) + throws IOException { + serveTeacher(); + final Path cache = Files.createDirectories(cacheRoot.resolve(CACHE_NAME)); + Files.write(cache.resolve(SENTENCEPIECE_MODEL), SENTENCEPIECE); + + HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null); + + assertTrue(Files.notExists(cache.resolve(SENTENCEPIECE_MODEL))); + } + + /** An explicit revision is downloaded, and pinned into a cache directory of its own. */ + @Test + void testAnExplicitRevisionIsRequestedAndCachedApart(@TempDir Path cacheRoot) throws IOException { + hub.serve(OTHER_COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_BLOB_SHA1), + OTHER_COMMIT); + hub.serve(OTHER_COMMIT, ModelFileNames.ONNX_MODEL, ONNX, quoted(ONNX_SHA256), OTHER_COMMIT); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID + "@" + OTHER_COMMIT, hub.base(), + cacheRoot, null); + + assertEquals(cacheRoot.resolve(cacheNameAt(OTHER_COMMIT)), cache); + assertEquals(OTHER_COMMIT, HuggingFaceModelCache.pinnedRevision(cache)); + assertTrue(hub.requests.stream() + .allMatch(p -> p.startsWith("/" + MODEL_ID + "/resolve/" + OTHER_COMMIT + "/")), + hub.requests.toString()); + } + + /** A named branch or tag is a revision too, and resolves to the commit the hub reports. */ + @Test + void testAnExplicitBranchIsResolvedToItsCommit(@TempDir Path cacheRoot) throws IOException { + hub.serve("refs-pr-1", ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_BLOB_SHA1)); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_BLOB_SHA1)); + hub.serve(COMMIT, ModelFileNames.ONNX_MODEL, ONNX, quoted(ONNX_SHA256)); + + final Path cache = HuggingFaceModelCache.resolve(MODEL_ID + "@refs-pr-1", hub.base(), + cacheRoot, null); + + assertEquals(COMMIT, HuggingFaceModelCache.pinnedRevision(cache)); + } + + @Test + void testARequestedCommitTheHubResolvesElsewhereIsRefused(@TempDir Path cacheRoot) { + hub.serve(OTHER_COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_BLOB_SHA1)); + + final IOException e = assertThrows(IOException.class, () -> HuggingFaceModelCache.resolve( + MODEL_ID + "@" + OTHER_COMMIT, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains("resolved to commit " + COMMIT), e.getMessage()); + } + + /** A directory recording one commit is not the answer to a reference naming another. */ + @Test + void testACacheRecordingAnotherCommitThanTheOneAskedForIsNotReused(@TempDir Path cacheRoot) + throws IOException { + final Path cache = Files.createDirectories(cacheRoot.resolve(cacheNameAt(OTHER_COMMIT))); + Files.createDirectories(cache.resolve(ModelFileNames.ONNX_MODEL).getParent()); + Files.write(cache.resolve(ModelFileNames.TOKENIZER_JSON), TOKENIZER); + Files.write(cache.resolve(ModelFileNames.ONNX_MODEL), ONNX); + Files.writeString(cache.resolve(HuggingFaceModelCache.REVISION_FILE), COMMIT + "\n"); + hub.serve(OTHER_COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_BLOB_SHA1), + OTHER_COMMIT); + hub.serve(OTHER_COMMIT, ModelFileNames.ONNX_MODEL, ONNX, quoted(ONNX_SHA256), OTHER_COMMIT); + + final Path resolved = HuggingFaceModelCache.resolve(MODEL_ID + "@" + OTHER_COMMIT, hub.base(), + cacheRoot, null); + + assertEquals(cache, resolved); + assertEquals(OTHER_COMMIT, HuggingFaceModelCache.pinnedRevision(resolved)); + assertFalse(hub.requests.isEmpty(), "the hub must be asked, not the stale record believed"); + } + + @Test + void testARevisionThatCannotBePinnedIsRefused(@TempDir Path cacheRoot) { + hub.reply(resolvePath(DEFAULT_REF, ModelFileNames.TOKENIZER_JSON), + new Reply(200, null, quoted(TOKENIZER_BLOB_SHA1), null, TOKENIZER)); + + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains("could not be pinned"), e.getMessage()); + assertTrue(Files.notExists(cacheRoot.resolve(CACHE_NAME))); + } + + @Test + void testAModelTheHubDoesNotHaveIsRefused(@TempDir Path cacheRoot) { + final IOException e = assertThrows(IOException.class, + () -> HuggingFaceModelCache.resolve(MODEL_ID, hub.base(), cacheRoot, null)); + + assertTrue(e.getMessage().contains("Failed to resolve revision 'main'"), e.getMessage()); + } + + /** + * Asserts that a failed download left nothing a distillation could pick up: neither the file it + * was verifying nor the temporary file it streamed into. + * + * @param cache The cache directory; need not exist. + * @param file The repository-relative name of the file that failed. + * @throws IOException Thrown if the directory cannot be walked. + */ + private void assertNothingUsable(Path cache, String file) throws IOException { + assertTrue(Files.notExists(cache.resolve(file)), file + " must not be published"); + if (Files.isDirectory(cache)) { + try (Stream entries = Files.walk(cache)) { + assertFalse(entries.anyMatch(p -> p.getFileName().toString().contains(".download")), + "a partial download must not be left behind"); + } + } + } + + /** Serves the ref and the two files a distillation needs, all at {@link #COMMIT}. */ + private void serveTeacher() { + hub.serve(DEFAULT_REF, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_BLOB_SHA1)); + hub.serve(COMMIT, ModelFileNames.TOKENIZER_JSON, TOKENIZER, quoted(TOKENIZER_BLOB_SHA1)); + hub.serve(COMMIT, ModelFileNames.ONNX_MODEL, ONNX, quoted(ONNX_SHA256)); + } + + /** + * {@return the request path of a file at a revision} + * + * @param revision The revision. + * @param file The repository-relative file name. + */ + private static String resolvePath(String revision, String file) { + return "/" + MODEL_ID + "/resolve/" + revision + "/" + file; + } + + /** + * {@return a header value in the quotes the hub puts around it} + * + * @param value The value. + */ + private static String quoted(String value) { + return "\"" + value + "\""; + } + + /** + * {@return the UTF-8 bytes of a fixture} + * + * @param content The content. + */ + private static byte[] bytes(String content) { + return content.getBytes(StandardCharsets.UTF_8); + } + + /** + * {@return a fixture of one character repeated, long enough to outrun a single read} + * + * @param content The character to repeat; must be an ASCII one, so that the fixture is as many + * bytes long as it is characters. + * @param length The number of characters. + */ + private static byte[] repeated(char content, int length) { + return bytes(String.valueOf(content).repeat(length)); + } + + /** + * One canned response. + * + * @param status The HTTP status. + * @param commit The {@code x-repo-commit} header value, or {@code null} to send none. + * @param etag The {@code x-linked-etag} header value, or {@code null} to send none. + * @param location The {@code Location} header value, or {@code null} to send none. + * @param body The response body, or {@code null} to send none. + */ + private record Reply(int status, String commit, String etag, String location, byte[] body) { + } + + /** + * A stand-in for the hub on the loopback interface, answering canned responses per request path + * and recording the paths it was asked for. + */ + private static final class Hub implements AutoCloseable { + + private final HttpServer server; + private final Map replies = new ConcurrentHashMap<>(); + private final List requests = Collections.synchronizedList(new ArrayList<>()); + + private Hub() throws IOException { + server = HttpServer.create(new InetSocketAddress(LOOPBACK, 0), 0); + server.createContext("/", this::answer); + server.start(); + } + + /** {@return the base URL of this hub, ending in a slash} */ + private String base() { + return "http://" + LOOPBACK + ":" + server.getAddress().getPort() + "/"; + } + + /** + * Serves a file at a revision, reporting {@link #COMMIT} as the commit the request resolved to. + * + * @param revision The revision to serve it at. + * @param file The repository-relative file name. + * @param body The response body. + * @param etag The {@code x-linked-etag} header value, or {@code null} to send none. + */ + private void serve(String revision, String file, byte[] body, String etag) { + serve(revision, file, body, etag, COMMIT); + } + + /** + * Serves a file at a revision. + * + * @param revision The revision to serve it at. + * @param file The repository-relative file name. + * @param body The response body. + * @param etag The {@code x-linked-etag} header value, or {@code null} to send none. + * @param commit The commit the request resolves to. + */ + private void serve(String revision, String file, byte[] body, String etag, String commit) { + reply(resolvePath(revision, file), new Reply(200, commit, etag, null, body)); + } + + /** + * Answers a file with a redirect carrying the headers, as the hub does for a file its content + * delivery network serves. + * + * @param revision The revision to serve it at. + * @param file The repository-relative file name. + * @param etag The {@code x-linked-etag} header value. + * @param target The path the redirect points at. + */ + private void redirect(String revision, String file, String etag, String target) { + reply(resolvePath(revision, file), new Reply(302, COMMIT, etag, target, null)); + } + + /** + * Answers a file with a status and nothing else. + * + * @param revision The revision to serve it at. + * @param file The repository-relative file name. + * @param status The HTTP status. + */ + private void status(String revision, String file, int status) { + reply(resolvePath(revision, file), new Reply(status, COMMIT, null, null, null)); + } + + /** + * Registers one canned response, replacing any response registered for the same path. + * + * @param path The request path. + * @param reply The response. + */ + private void reply(String path, Reply reply) { + replies.put(path, reply); + } + + /** + * Answers one request, with 404 when nothing is registered for its path. + * + * @param exchange The exchange. + * @throws IOException Thrown if the response headers cannot be sent. + */ + private void answer(HttpExchange exchange) throws IOException { + final String path = exchange.getRequestURI().getPath(); + requests.add(path); + final Reply reply = replies.get(path); + if (reply == null) { + exchange.sendResponseHeaders(NOT_FOUND, -1); + exchange.close(); + return; + } + if (reply.commit() != null) { + exchange.getResponseHeaders().add("x-repo-commit", reply.commit()); + } + if (reply.etag() != null) { + exchange.getResponseHeaders().add("x-linked-etag", reply.etag()); + } + if (reply.location() != null) { + exchange.getResponseHeaders().add("Location", reply.location()); + } + if (reply.body() == null) { + exchange.sendResponseHeaders(reply.status(), -1); + } else { + exchange.sendResponseHeaders(reply.status(), reply.body().length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(reply.body()); + } catch (IOException e) { + // The client closes a body it does not need, which fails this write; that is the point + // of the header-only requests, so it is not a test failure. + } + } + exchange.close(); + } + + @Override + public void close() { + server.stop(0); + } + } + /** + * Verifies that references differing only in a character the readable part of the directory + * name flattens do not share a cache directory. They did: the name replaced '.', '@' and '/' + * without disambiguating them, so three distinct teachers mapped to one directory, and the + * cached fast path would answer from it without contacting the hub. + */ + @Test + void testDistinctTeachersDoNotShareACacheDirectory() throws Exception { + final java.util.Set names = new java.util.HashSet<>(); + for (final String teacher : java.util.List.of( + "acme/model_v1", "acme/model.v1", "acme/model@v1", "acme/model-v1")) { + names.add(HuggingFaceModelCache.cacheDirectoryName(teacher)); + } + assertEquals(4, names.size(), "each distinct teacher reference needs its own directory"); + } + } From 7694191addc1098a6a7cd8db93f71388289075c0 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 2 Aug 2026 07:40:17 -0400 Subject: [PATCH 57/82] OPENNLP-1877: Batch same-length inputs into one ONNX run in embedAll The default TextEmbedder.embedAll embeds one text at a time, so a document pipeline paid one session run per sentence or token span. SentenceVectorsDL now tokenizes the batch up front, groups inputs by tokenized length, and runs each group through the session once with shape [group size, length]. Grouping by length means a batch never pads: every row is computed from exactly the tensors its single-input call would have used, so results match the per-text calls bit for bit, which the new test pins with exact float equality over mixed-length inputs. --- .../opennlp/dl/vectors/SentenceVectorsDL.java | 97 ++++++++++++++++++- .../SentenceVectorsDLEmbedderTest.java | 35 ++++++- 2 files changed, 129 insertions(+), 3 deletions(-) diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java index 8069517de3..2bab92e0ef 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/SentenceVectorsDL.java @@ -20,9 +20,11 @@ import java.io.File; import java.io.IOException; import java.nio.LongBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import ai.onnxruntime.NodeInfo; @@ -62,8 +64,9 @@ * is called; callers must not race {@code close()} with inference methods.

* *

{@link #getVectors(String)} is the primary entry point; {@link #embed(CharSequence)} - * adapts it to the {@link TextEmbedder} contract. The inherited {@code embedAll} embeds one - * text at a time.

+ * adapts it to the {@link TextEmbedder} contract. {@link #embedAll(List)} runs one batched + * session per distinct tokenized length, so a batch of same-length inputs costs one + * inference instead of one per input.

*/ @ThreadSafe public class SentenceVectorsDL extends AbstractDL implements TextEmbedder { @@ -168,6 +171,96 @@ public float[] embed(final CharSequence text) { } } + /** + * {@inheritDoc} + * + *

Batched execution: the inputs are tokenized up front, grouped by tokenized length, + * and each group runs through the session once with shape {@code [group size, length]}. + * Grouping by length means a batch never pads, so every row is computed from exactly the + * tensors its single-input call would have used. A length group of one executes the + * same {@code [1, length]} shapes as {@link #getVectors(String)}.

+ * + * @throws IllegalArgumentException Thrown if {@code texts} is {@code null} or contains + * {@code null}. + * @throws IllegalStateException Thrown if inference fails; the cause carries the + * underlying {@link OrtException}. + */ + @Override + public float[][] embedAll(final List texts) { + if (texts == null) { + throw new IllegalArgumentException("Texts must not be null"); + } + final float[][] vectors = new float[texts.size()][]; + if (texts.isEmpty()) { + return vectors; + } + final Tokens[] encoded = new Tokens[texts.size()]; + final Map> byLength = new HashMap<>(); + for (int i = 0; i < texts.size(); i++) { + final CharSequence text = texts.get(i); + if (text == null) { + throw new IllegalArgumentException("Texts must not contain null"); + } + encoded[i] = tokenize(text instanceof String s ? s : text.toString(), tokenizer, vocab); + byLength.computeIfAbsent(encoded[i].ids().length, length -> new ArrayList<>()).add(i); + } + try { + for (final List group : byLength.values()) { + runBatch(encoded, group, vectors); + } + } catch (OrtException e) { + throw new IllegalStateException("Sentence vector inference failed.", e); + } + return vectors; + } + + /** + * Runs one inference over a group of same-length encodings and stores each row's + * {@code [CLS]}-position vector under its original input index. + * + * @param encoded The tokenized inputs, indexed by input position. + * @param group The input positions sharing one tokenized length, in input order. + * @param vectors The output array to fill, indexed by input position. + * @throws OrtException Thrown if an error occurs during inference. + */ + private void runBatch(final Tokens[] encoded, final List group, + final float[][] vectors) throws OrtException { + + final int batch = group.size(); + final int length = encoded[group.get(0)].ids().length; + final long[] ids = new long[batch * length]; + final long[] mask = new long[batch * length]; + final long[] types = new long[batch * length]; + for (int b = 0; b < batch; b++) { + final Tokens tokens = encoded[group.get(b)]; + System.arraycopy(tokens.ids(), 0, ids, b * length, length); + System.arraycopy(tokens.mask(), 0, mask, b * length, length); + System.arraycopy(tokens.types(), 0, types, b * length, length); + } + + final Map inputs = new HashMap<>(); + final long[] shape = {batch, length}; + + try { + inputs.put(INPUT_IDS, OnnxTensor.createTensor(env, LongBuffer.wrap(ids), shape)); + + inputs.put(ATTENTION_MASK, OnnxTensor.createTensor(env, LongBuffer.wrap(mask), shape)); + + inputs.put(TOKEN_TYPE_IDS, OnnxTensor.createTensor(env, LongBuffer.wrap(types), shape)); + + try (OrtSession.Result result = session.run(inputs)) { + // getValue() copies the tensor into Java arrays, so the result can be closed safely. + final float[][][] v = (float[][][]) result.get(0).getValue(); + for (int b = 0; b < batch; b++) { + vectors[group.get(b)] = v[b][0]; + } + } + } finally { + inputs.values().forEach(OnnxTensor::close); + } + + } + /** * {@inheritDoc} * diff --git a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java index 06fd847cea..306f38595f 100644 --- a/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java +++ b/opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java @@ -23,6 +23,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -84,7 +85,7 @@ void testEmbedderContractOverARealSession(@TempDir Path dir) throws Exception { assertArrayEquals(CLS_VECTOR, embedder.embed("hello world"), 1e-5f); assertArrayEquals(CLS_VECTOR, embedder.embed(new StringBuilder("hello world")), 1e-5f); - // The inherited default batch method returns one vector per input, in input order; + // The batch method returns one vector per input, in input order; // this model's [CLS]-position output is input-independent by construction. final float[][] batch = embedder.embedAll(List.of("hello world", "hello")); assertEquals(2, batch.length); @@ -95,4 +96,36 @@ void testEmbedderContractOverARealSession(@TempDir Path dir) throws Exception { assertThrows(IllegalArgumentException.class, () -> embedder.embedAll(null)); } } + + /** + * Drives the batched path over inputs of mixed tokenized lengths ("hello" encodes one + * token shorter than "hello world") and asserts every row reproduces its single-input + * vector exactly: the length-grouped batch never pads, so the computation per row is + * the computation the single call performs. + */ + @Test + void testEmbedAllMatchesSingleEmbedsExactly(@TempDir Path dir) throws Exception { + try (SentenceVectorsDL vectors = new SentenceVectorsDL(model(dir), vocab(dir))) { + final List texts = List.of("hello", "hello world", "world", "hello world", + "hello"); + final float[][] batch = vectors.embedAll(texts); + assertEquals(texts.size(), batch.length); + for (int i = 0; i < texts.size(); i++) { + assertArrayEquals(vectors.embed(texts.get(i)), batch[i]); + } + } + } + + /** + * Asserts the batch contract edges: an empty input yields an empty batch, and a + * {@code null} element is rejected rather than failing later inside the session. + */ + @Test + void testEmbedAllEdges(@TempDir Path dir) throws Exception { + try (SentenceVectorsDL vectors = new SentenceVectorsDL(model(dir), vocab(dir))) { + assertEquals(0, vectors.embedAll(List.of()).length); + assertThrows(IllegalArgumentException.class, + () -> vectors.embedAll(Arrays.asList("hello", null))); + } + } } From 742148c5ef35216f8c0e91c8d45a6c727afedb82 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 6 Aug 2026 08:30:52 -0400 Subject: [PATCH 58/82] OPENNLP-1877: Mark public API experimental and reject bare minus headers Annotate the public embeddings types with @Experimental, note that in the manual, and fail loud on a lone "-" in a skipped safetensors field. --- opennlp-docs/src/docbkx/embeddings.xml | 6 ++++++ .../src/main/java/opennlp/embeddings/Neighbor.java | 5 +++++ .../src/main/java/opennlp/embeddings/SafetensorsFile.java | 4 ++++ .../java/opennlp/embeddings/StaticEmbeddingModel.java | 4 ++++ .../src/main/java/opennlp/embeddings/TensorInfo.java | 5 +++++ .../opennlp/embeddings/SafetensorsHeaderParserTest.java | 8 ++++++++ 6 files changed, 32 insertions(+) diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index d28b1d1b97..cdc83d935b 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -49,6 +49,12 @@ No model is bundled with the module. Callers point it at a model directory they downloaded; the table's own license applies to the table. + + The public API of this module + (StaticEmbeddingModel, SafetensorsFile, + TensorInfo, and Neighbor) is experimental and may change + in a later release. +
diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java index d136f1703d..e5f7a02cb1 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Neighbor.java @@ -16,13 +16,18 @@ */ package opennlp.embeddings; +import opennlp.tools.util.java.Experimental; + /** * One vocabulary token found near a query vector by {@link StaticEmbeddingModel#mostSimilar} * or {@link StaticEmbeddingModel#analogy}, most similar first. * + *

Warning: Experimental new feature; the API might change in a later release.

+ * * @param token The vocabulary token: one subword piece of the model's tokenizer, which is * not necessarily a whole word. * @param similarity Cosine similarity to the query vector, in {@code [-1, 1]}. */ +@Experimental public record Neighbor(String token, double similarity) { } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 316c71770e..0832e9be92 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -31,6 +31,7 @@ import java.util.Set; import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.java.Experimental; /** * Reads a safetensors file: an 8-byte @@ -47,7 +48,10 @@ * *

Instances are immutable and safe for concurrent use: every {@link #readFloats(String)} * call opens its own channel and decodes into a fresh array the caller owns.

+ * + *

Warning: Experimental new feature; the API might change in a later release.

*/ +@Experimental @ThreadSafe public final class SafetensorsFile { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index bd01721242..b2c0adc600 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -34,6 +34,7 @@ import opennlp.tools.tokenize.SubwordTokenizer; import opennlp.tools.tokenize.WordpieceEncoder; import opennlp.tools.tokenize.WordpieceTokenizer; +import opennlp.tools.util.java.Experimental; /** * A static (non-contextual) sentence embedding model: a per-token vector table plus subword @@ -55,7 +56,10 @@ * zero vector.

* *

Instances are immutable and safe for concurrent use after construction.

+ * + *

Warning: Experimental new feature; the API might change in a later release.

*/ +@Experimental @ThreadSafe public final class StaticEmbeddingModel implements TextEmbedder { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java index 32b767052c..a083b4026a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TensorInfo.java @@ -18,10 +18,14 @@ import java.util.Arrays; +import opennlp.tools.util.java.Experimental; + /** * Header metadata for one tensor in a safetensors file, as declared by the file's own JSON * header. Carries no data; {@link SafetensorsFile#readFloats(String)} resolves the bytes. * + *

Warning: Experimental new feature; the API might change in a later release.

+ * * @param name The tensor's name, the key it was declared under. Never {@code null}. * @param dtype The declared element type (e.g. {@code "F32"}, {@code "F16"}, * {@code "I64"}), exactly as written in the header. Never {@code null}. @@ -31,6 +35,7 @@ * of the header, not the start of the file). * @param dataOffsetEnd End byte offset (exclusive) into the data section. */ +@Experimental public record TensorInfo(String name, String dtype, int[] shape, long dataOffsetBegin, long dataOffsetEnd) { diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java index 2e25beec0c..7c61709929 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java @@ -172,6 +172,14 @@ void testMalformedNumberInSkippedFieldFailsLoudly() { assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(header)); } + @Test + void testLoneMinusInSkippedFieldFailsLoudly() { + // A bare "-" is not a JSON number; the skip path must reject it rather than treating it as one. + final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]," + + "\"data_offsets\":[0,4],\"unknown\":-}}"; + assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(header)); + } + @Test void testWellFormedNumbersInSkippedFieldsAreAccepted() { final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]," From 595959dc64fe1d5fc4a0cd8580fb7a9f78ec7604 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sat, 8 Aug 2026 18:56:14 -0400 Subject: [PATCH 59/82] OPENNLP-1877: Address review: complete experimental markers, hrefs, and edge tests Annotate ModelAssembler, ModelDistiller, and TextEmbedder with @Experimental like the other public embeddings types and extend the manual's note. Link Model2Vec and the randomized SVD paper at first mention. Make the usage example test mirror the manual's directory-load listing. Cover whitespace-only and supplementary-plane input in the embed path. Document the Checksum constructor. --- .../tools/embeddings/TextEmbedder.java | 5 +++ opennlp-docs/src/docbkx/embeddings.xml | 5 +-- .../embeddings/HuggingFaceModelCache.java | 7 ++++ .../opennlp/embeddings/ModelAssembler.java | 11 ++++-- .../opennlp/embeddings/ModelDistiller.java | 5 +++ .../embeddings/OnnxTeacherEncoder.java | 4 ++- .../opennlp/embeddings/RandomizedPca.java | 3 +- .../embeddings/StaticEmbeddingModel.java | 4 ++- .../opennlp/embeddings/TeacherTokenizer.java | 3 +- .../embeddings/EmbeddingTestFixtures.java | 34 ++++++++++++++++--- .../embeddings/StaticEmbeddingModelTest.java | 22 ++++++++++++ .../StaticEmbeddingUsageExampleTest.java | 13 ++++--- 12 files changed, 96 insertions(+), 20 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java index b6b4e880f2..32b3396afa 100644 --- a/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java +++ b/opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java @@ -18,6 +18,8 @@ import java.util.List; +import opennlp.tools.util.java.Experimental; + /** * Encodes a piece of text into a single fixed-length vector. * @@ -29,7 +31,10 @@ * *

Thread safety is implementation specific. Failures during encoding surface as unchecked * exceptions carrying the underlying cause.

+ * + *

Warning: Experimental new feature; the API might change in a later release.

*/ +@Experimental public interface TextEmbedder { /** diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index cdc83d935b..227acf6073 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -52,8 +52,9 @@ The public API of this module (StaticEmbeddingModel, SafetensorsFile, - TensorInfo, and Neighbor) is experimental and may change - in a later release. + TensorInfo, Neighbor, ModelDistiller, and + ModelAssembler), together with the TextEmbedder interface + in opennlp-api, is experimental and may change in a later release.
diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java index 18f0abc09a..1e61977fc7 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java @@ -561,6 +561,13 @@ private enum Checksum { private final String algorithm; private final int hexLength; + /** + * Creates a digest form. + * + * @param displayName The name used in error messages. + * @param algorithm The {@link java.security.MessageDigest} algorithm name. + * @param hexLength The length of the digest's hex form. + */ Checksum(String displayName, String algorithm, int hexLength) { this.displayName = displayName; this.algorithm = algorithm; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java index e8d4bba9c9..93360d2891 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java @@ -26,9 +26,13 @@ import java.util.List; import java.util.Map; +import opennlp.tools.util.java.Experimental; + /** - * Turns a distilled model directory (the layout the Model2Vec {@code save_pretrained} writes) into - * a directory {@link StaticEmbeddingModel#load(Path)} can open, then verifies it by loading it. + * Turns a distilled model directory (the layout the + * Model2Vec {@code save_pretrained} writes) + * into a directory {@link StaticEmbeddingModel#load(Path)} can open, then verifies it by loading + * it. * *

A distillation ships {@code model.safetensors}, {@code tokenizer.json}, and * {@code config.json}, but not the two files the loader also needs for a WordPiece model @@ -40,7 +44,10 @@ * *

Assembly writes only the missing files and never overwrites an existing one, so a directory * a caller already completed by hand is left intact.

+ * + *

Warning: Experimental new feature; the API might change in a later release.

*/ +@Experimental public final class ModelAssembler { /** The WordPiece tokenizer family, the {@code model.type} of a BERT-style distillation. */ diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java index a602cbe43a..18a8a87aa3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java @@ -21,6 +21,8 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import opennlp.tools.util.java.Experimental; + /** * Distills a sentence-transformer teacher into a static embedding table in the layout * {@link StaticEmbeddingModel#load(Path)} opens, reproducing @@ -46,7 +48,10 @@ *

The teacher directory must hold {@code tokenizer.json} and {@code onnx/model.onnx} (the * ONNX export every sentence-transformer ships on the Hugging Face hub); a local * {@code tokenizer_config.json} supplies the pad token when present.

+ * + *

Warning: Experimental new feature; the API might change in a later release.

*/ +@Experimental public final class ModelDistiller { /** Model2Vec's default SIF coefficient for the Zipf weighting. */ diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java index a1df7692db..7045702f56 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/OnnxTeacherEncoder.java @@ -34,7 +34,9 @@ /** * Runs a teacher transformer over id sequences through its ONNX graph and mean-pools the last - * hidden states, the forward pass Model2Vec's distillation performs per vocabulary token. The + * hidden states, the forward pass + * Model2Vec's distillation performs per + * vocabulary token. The * graph is fed exactly the inputs it declares: {@code input_ids} and {@code attention_mask} for * every model, plus a zero {@code token_type_ids} for the BERT-family graphs that ask for one. * The pooled output is the mean of the single rank-3 float output (the diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java index 56a2b095c4..f64d9bde96 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/RandomizedPca.java @@ -21,7 +21,8 @@ import java.util.stream.IntStream; /** - * Principal component analysis by randomized SVD (Halko, Martinsson, Tropp), the approximation + * Principal component analysis by randomized SVD + * (Halko, Martinsson, Tropp), the approximation * Model2Vec's distillation performs with a dense LAPACK SVD through scikit-learn. A dense SVD of * a vocabulary-size matrix (250k rows for a multilingual teacher) is not practical in pure Java, * so the top components are found with a random range finder and {@value #POWER_ITERATIONS} power diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index b2c0adc600..df93b6d7d7 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -41,7 +41,9 @@ * tokenization. Embedding a sentence is tokenize, gather each piece's row, optionally weight, * mean-pool, and optionally L2-normalize; there is no model forward pass. * - *

It loads distilled tables in the Model2Vec release layout for both tokenizer families: + *

It loads distilled tables in the + * Model2Vec release layout for both + * tokenizer families: * WordPiece models carry a {@code vocab.txt} whose line number is the matrix row, and * SentencePiece models carry a Unigram {@code tokenizer.json} whose {@code model.vocab} list * order is the row order, next to the trained SentencePiece {@code .model} file that performs diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java index 1602df3a56..dd9dc095c6 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java @@ -29,7 +29,8 @@ import java.util.regex.Pattern; /** - * The tokenizer side of a teacher model, distilled the way Model2Vec distills it. The class reads + * The tokenizer side of a teacher model, distilled the way + * Model2Vec distills it. The class reads * the teacher's {@code tokenizer.json} (and, when present, its {@code tokenizer_config.json} for * the pad token), decides which vocabulary rows survive into the static table, and rewrites the * {@code tokenizer.json} so it describes the distilled table. diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java index d7f117b939..53e1ffce46 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java @@ -65,12 +65,36 @@ private EmbeddingTestFixtures() { */ static StaticEmbeddingModel loadAnalogyModel(Path dir, Normalization normalization) throws IOException { - final Path vocabulary = dir.resolve("vocab.txt"); - Files.write(vocabulary, ANALOGY_VOCABULARY); - final Path safetensors = dir.resolve("model.safetensors"); - SafetensorsTestFiles.write(safetensors, + writeVocabularyAndMatrix(dir); + return StaticEmbeddingModel.load(dir.resolve("vocab.txt"), dir.resolve("model.safetensors"), + Casing.UNCASED, normalization); + } + + /** + * Writes {@link #ANALOGY_VOCABULARY} and {@link #ANALOGY_ROWS} into a directory as a complete + * WordPiece model directory (with its two JSON configuration files), so a test can load it + * with {@code StaticEmbeddingModel.load(Path)} the way the manual's usage listing shows. + * + * @param dir The directory to write the model files into. + * @throws IOException Thrown if writing a fixture file fails. + */ + static void writeAnalogyDirectory(Path dir) throws IOException { + writeVocabularyAndMatrix(dir); + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false}"); + Files.writeString(dir.resolve("tokenizer_config.json"), "{\"do_lower_case\":true}"); + } + + /** + * Writes the analogy table's {@code vocab.txt} and {@code model.safetensors} into a directory. + * + * @param dir The directory to write the fixture files into. + * @throws IOException Thrown if writing a fixture file fails. + */ + private static void writeVocabularyAndMatrix(Path dir) throws IOException { + Files.write(dir.resolve("vocab.txt"), ANALOGY_VOCABULARY); + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), SafetensorsTestFiles.matrix("embeddings", ANALOGY_ROWS)); - return StaticEmbeddingModel.load(vocabulary, safetensors, Casing.UNCASED, normalization); } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index 9b612bbba8..eed96e783f 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -213,6 +213,28 @@ void testEmbedOfEmptyTextIsZeroVectorNotAnError(@TempDir Path dir) throws IOExce assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed(""), 1e-5f); } + @Test + void testEmbedOfWhitespaceOnlyTextIsZeroVector(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); + + // Whitespace-only text produces no content pieces at all, unlike unknown text, which still + // produces a (skipped) [UNK]; both must pool to the zero vector without dividing by zero. + assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed(" \t\n "), 1e-5f); + } + + @Test + void testEmbedSkipsSupplementaryPlaneTextAsUnknown(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = + StaticEmbeddingModel.load(writeVocab(dir), writeSafetensors(dir, false), + Casing.UNCASED, Normalization.NONE); + + // An emoji is a supplementary-plane character (a surrogate pair in Java) no vocabulary + // piece covers; it must fold to [UNK] and be skipped, leaving just "cat" in the pool. + assertArrayEquals(new float[] {5f, 50f, 500f}, model.embed("cat \uD83D\uDE00"), 1e-5f); + } + @Test void testDimensionAndVocabularySizeAccessors(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java index 3e14d1a6e2..ae8278ce31 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java @@ -23,22 +23,21 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import opennlp.embeddings.StaticEmbeddingModel.Normalization; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Pins the cookbook path documented in {@code embeddings.xml}: load a - * {@link StaticEmbeddingModel}, embed a sentence, and call {@code similarity}, - * {@code mostSimilar}, and {@code analogy}. + * Pins the cookbook path documented in {@code embeddings.xml}, mirroring its usage listing: + * load a model directory with {@link StaticEmbeddingModel#load(Path)}, embed a text, and call + * {@code similarity}, {@code mostSimilar}, and {@code analogy}. */ public class StaticEmbeddingUsageExampleTest { @Test void testEmbedSimilarityNeighborsAndAnalogy(@TempDir Path dir) throws IOException { - final StaticEmbeddingModel model = - EmbeddingTestFixtures.loadAnalogyModel(dir, Normalization.NONE); + EmbeddingTestFixtures.writeAnalogyDirectory(dir); + + final StaticEmbeddingModel model = StaticEmbeddingModel.load(dir); final float[] vector = model.embed("king"); assertEquals(2, vector.length); From 71b16b391f1496be669c27feb18c95d7bbbec090 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:43:03 -0400 Subject: [PATCH 60/82] OPENNLP-1877: Throw InvalidFormatException for malformed model content Malformed model content (malformed safetensors headers and config files, dimension and row-count disagreements, ambiguous matrix sources, bare minus headers) now throws the checked opennlp.tools.util .InvalidFormatException across StaticEmbeddingModel.load and every loader it calls, matching SentencePieceTokenizer.load. IllegalArgumentException stays for caller argument errors only. The CLI tools report the new type with the same exit code as before. --- .../embeddings/EmbeddingVocabulary.java | 17 +++--- .../opennlp/embeddings/FlatJsonFields.java | 18 ++++-- .../java/opennlp/embeddings/JsonCursor.java | 42 +++++++------- .../opennlp/embeddings/ModelAssembler.java | 30 ++++++---- .../opennlp/embeddings/SafetensorsFile.java | 50 +++++++++------- .../embeddings/SafetensorsHeaderParser.java | 21 ++++--- .../embeddings/StaticEmbeddingModel.java | 57 +++++++++++-------- .../opennlp/embeddings/TeacherTokenizer.java | 55 +++++++++++------- .../embeddings/TokenizerJsonVocab.java | 33 +++++++---- .../embeddings/cmdline/AssembleModelTool.java | 3 +- .../embeddings/cmdline/DistillModelTool.java | 3 +- .../embeddings/EmbeddingVocabularyTest.java | 10 ++-- .../embeddings/FlatJsonFieldsTest.java | 14 +++-- .../embeddings/SafetensorsFileTest.java | 34 +++++------ .../SafetensorsHeaderParserTest.java | 26 +++++---- ...StaticEmbeddingModelSentencePieceTest.java | 13 +++-- .../embeddings/StaticEmbeddingModelTest.java | 21 ++++--- .../embeddings/TeacherTokenizerTest.java | 14 +++-- .../embeddings/TokenizerJsonVocabTest.java | 22 +++---- 19 files changed, 280 insertions(+), 203 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java index 98b2c698fc..205c0980a6 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingVocabulary.java @@ -26,6 +26,7 @@ import java.util.Set; import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.InvalidFormatException; /** * The row table of a static embedding matrix: piece string to row index and back. Row {@code id} @@ -54,8 +55,8 @@ private EmbeddingVocabulary(Map idByToken, List tokenBy * * @param file The vocabulary file. Must not be {@code null} and must exist. * @return The parsed vocabulary. - * @throws IllegalArgumentException Thrown if {@code file} is {@code null}, missing, or - * contains a duplicate token. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing. + * @throws InvalidFormatException Thrown if the file contains a duplicate token. * @throws IOException Thrown if reading the file fails. */ static EmbeddingVocabulary fromVocabTxt(Path file) throws IOException { @@ -69,8 +70,9 @@ static EmbeddingVocabulary fromVocabTxt(Path file) throws IOException { * * @param file The {@code tokenizer.json} file. Must not be {@code null} and must exist. * @return The parsed vocabulary. - * @throws IllegalArgumentException Thrown if {@code file} is {@code null}, missing, or not a - * well-formed Unigram {@code tokenizer.json}, or a piece appears more than once. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing. + * @throws InvalidFormatException Thrown if the file is not a well-formed Unigram + * {@code tokenizer.json} or a piece appears more than once. * @throws IOException Thrown if reading the file fails. */ static EmbeddingVocabulary fromTokenizerJson(Path file) throws IOException { @@ -100,14 +102,15 @@ private static void requireRegularFile(Path file) { * @param lines The tokens, one per element; the index is the token's row. * @param sourceName The source's name, for error messages. * @return The parsed vocabulary. - * @throws IllegalArgumentException Thrown if a token appears more than once. + * @throws InvalidFormatException Thrown if a token appears more than once. */ - static EmbeddingVocabulary fromLines(List lines, String sourceName) { + static EmbeddingVocabulary fromLines(List lines, String sourceName) + throws InvalidFormatException { final Map idByToken = new LinkedHashMap<>(lines.size() * 2); for (int id = 0; id < lines.size(); id++) { final String token = lines.get(id); if (idByToken.putIfAbsent(token, id) != null) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "Vocabulary " + sourceName + " declares token '" + token + "' more than once, at rows " + idByToken.get(token) + " and " + id); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java index 11d1556722..07511bcaac 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FlatJsonFields.java @@ -20,6 +20,8 @@ import java.nio.file.Files; import java.nio.file.Path; +import opennlp.tools.util.InvalidFormatException; + /** * Reads single top-level fields out of a small flat JSON configuration file (a model's * {@code config.json} or {@code tokenizer_config.json}) without a JSON library dependency. Only @@ -43,7 +45,8 @@ private FlatJsonFields() { * @param field The top-level field name to read. Must not be {@code null}. * @return The field's value, or {@code null} when the field is absent or explicitly JSON * {@code null} (the formats treat those the same: fall back to the default). - * @throws IllegalArgumentException Thrown if the file is not a well-formed JSON object, the + * @throws IllegalArgumentException Thrown if an argument is {@code null}. + * @throws InvalidFormatException Thrown if the file is not a well-formed JSON object, the * field appears more than once, or its value is neither a boolean nor {@code null}. * @throws IOException Thrown if reading the file fails. */ @@ -70,7 +73,8 @@ static Boolean topLevelBoolean(Path file, String field) throws IOException { * @param field The top-level field name to read. Must not be {@code null}. * @return The field's value, or {@code null} when the field is absent or explicitly JSON * {@code null} (the formats treat those the same: fall back to the default). - * @throws IllegalArgumentException Thrown if the file is not a well-formed JSON object, the + * @throws IllegalArgumentException Thrown if an argument is {@code null}. + * @throws InvalidFormatException Thrown if the file is not a well-formed JSON object, the * field appears more than once, or its value is neither a string nor {@code null}. * @throws IOException Thrown if reading the file fails. */ @@ -96,8 +100,9 @@ static String topLevelString(Path file, String field) throws IOException { * @param valueReader Reads the matched field's value off the cursor. * @param The value type the reader produces. * @return The field's value, or {@code null} when the field is absent. - * @throws IllegalArgumentException Thrown if an argument is {@code null}, the file is not a - * well-formed JSON object, or the field appears more than once. + * @throws IllegalArgumentException Thrown if an argument is {@code null}. + * @throws InvalidFormatException Thrown if the file is not a well-formed JSON object or the + * field appears more than once. * @throws IOException Thrown if reading the file fails. */ private static T topLevelField(Path file, String field, ValueReader valueReader) @@ -161,8 +166,9 @@ private interface ValueReader { * * @param cursor The cursor, positioned at the value's first character. * @return The decoded value, or {@code null} for a JSON {@code null}. - * @throws IllegalArgumentException Thrown if the value is not of the expected type. + * @throws InvalidFormatException Thrown if the value is malformed or not of the expected + * type. */ - T read(JsonCursor cursor); + T read(JsonCursor cursor) throws InvalidFormatException; } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java index 91c3fba0f3..236a9aed0a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java @@ -16,6 +16,8 @@ */ package opennlp.embeddings; +import opennlp.tools.util.InvalidFormatException; + /** * Cursor primitives shared by this package's purpose-built JSON readers * ({@link SafetensorsHeaderParser}, {@link FlatJsonFields}, {@link TokenizerJsonVocab}, @@ -23,7 +25,9 @@ * value of any type. Deliberately not a general JSON * library: no floating-point decoding, no document model; each reader drives the cursor over * its own known-shape input and fails loud on anything else, with the input's name and the - * offending offset in every message. + * offending offset in every message. Malformed input is a checked + * {@link InvalidFormatException}, the exception model content errors carry throughout this + * package. */ final class JsonCursor { @@ -58,9 +62,9 @@ int position() { /** * {@return the character at the cursor without advancing} * - * @throws IllegalArgumentException Thrown if the cursor is at the end of the input. + * @throws InvalidFormatException Thrown if the cursor is at the end of the input. */ - char peek() { + char peek() throws InvalidFormatException { if (position >= text.length()) { throw malformed("Unexpected end of input"); } @@ -70,9 +74,9 @@ char peek() { /** * {@return the character at the cursor, advancing past it} * - * @throws IllegalArgumentException Thrown if the cursor is at the end of the input. + * @throws InvalidFormatException Thrown if the cursor is at the end of the input. */ - char consume() { + char consume() throws InvalidFormatException { final char c = peek(); position++; return c; @@ -82,9 +86,9 @@ char consume() { * Consumes the next character, requiring it to be {@code c}. * * @param c The expected character. - * @throws IllegalArgumentException Thrown if the next character is not {@code c}. + * @throws InvalidFormatException Thrown if the next character is not {@code c}. */ - void expect(char c) { + void expect(char c) throws InvalidFormatException { final char actual = consume(); if (actual != c) { throw malformed("Expected '" + c + "', got '" + actual + "'"); @@ -110,9 +114,9 @@ boolean consumeLiteral(String literal) { * Requires the rest of the input to be whitespace only. * * @param message What to report when other content follows. - * @throws IllegalArgumentException Thrown if non-whitespace content follows the cursor. + * @throws InvalidFormatException Thrown if non-whitespace content follows the cursor. */ - void requireEnd(String message) { + void requireEnd(String message) throws InvalidFormatException { skipWhitespace(); if (position < text.length()) { throw malformed(message); @@ -122,9 +126,9 @@ void requireEnd(String message) { /** * {@return the JSON string starting at the cursor, with escapes decoded} * - * @throws IllegalArgumentException Thrown if the string is unterminated or has a bad escape. + * @throws InvalidFormatException Thrown if the string is unterminated or has a bad escape. */ - String parseString() { + String parseString() throws InvalidFormatException { expect('"'); final StringBuilder value = new StringBuilder(); while (true) { @@ -144,7 +148,7 @@ String parseString() { } /** {@return the character named by the escape sequence following a backslash} */ - private char parseEscape() { + private char parseEscape() throws InvalidFormatException { if (position >= text.length()) { throw malformed("Unterminated escape sequence"); } @@ -164,7 +168,7 @@ private char parseEscape() { } /** {@return the character named by a {@code \\uXXXX} escape} */ - private char parseUnicodeEscape() { + private char parseUnicodeEscape() throws InvalidFormatException { if (position + 4 > text.length()) { throw malformed("Truncated \\u escape sequence"); } @@ -187,7 +191,7 @@ private char parseUnicodeEscape() { * Skips one JSON number, holding it to the grammar (optional minus, digits, optional fraction, * optional signed exponent) so malformed input fails loud even in a skipped field. */ - private void skipNumber() { + private void skipNumber() throws InvalidFormatException { if (peek() == '-') { position++; } @@ -225,9 +229,9 @@ private void skipNumber() { /** * {@return the integer starting at the cursor, parsed as a {@code long}} * - * @throws IllegalArgumentException Thrown if no integer is present or it overflows a long. + * @throws InvalidFormatException Thrown if no integer is present or it overflows a long. */ - long parseLong() { + long parseLong() throws InvalidFormatException { final int start = position; if (peek() == '-') { position++; @@ -249,7 +253,7 @@ long parseLong() { * Skips one JSON value of any type (string, number, array, object, true/false/null), so a * reader tolerates fields it does not care about. */ - void skipValue() { + void skipValue() throws InvalidFormatException { skipWhitespace(); final char c = peek(); if (c == '"') { @@ -309,8 +313,8 @@ void skipValue() { * * @param message What was wrong at the cursor. */ - IllegalArgumentException malformed(String message) { - return new IllegalArgumentException( + InvalidFormatException malformed(String message) { + return new InvalidFormatException( "Malformed " + inputName + " at offset " + position + ": " + message); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java index 93360d2891..74cdfbf72c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.util.java.Experimental; /** @@ -85,8 +86,10 @@ public record Result(String family, int dimension, int vocabularySize, * {@code tokenizer.json}, and {@code config.json}. * @return The assembly result. * @throws IllegalArgumentException Thrown if {@code modelDirectory} is {@code null}, is not a - * directory, is missing a required distillation file, is a SentencePiece model without its - * {@code .model} file, or does not load after assembly. + * directory, is missing a required distillation file, or is a SentencePiece model without + * its {@code .model} file. + * @throws InvalidFormatException Thrown if a model file is malformed, its tokenizer family is + * unsupported, or the directory does not load after assembly. * @throws IOException Thrown if reading or writing a file fails. */ public static Result assemble(Path modelDirectory) throws IOException { @@ -105,7 +108,7 @@ public static Result assemble(Path modelDirectory) throws IOException { return switch (tokenizer.modelType()) { case FAMILY_WORDPIECE -> assembleWordpiece(modelDirectory, tokenizer); case FAMILY_UNIGRAM -> assembleSentencePiece(modelDirectory); - default -> throw new IllegalArgumentException(tokenizerJson + " has a '" + default -> throw new InvalidFormatException(tokenizerJson + " has a '" + tokenizer.modelType() + "' tokenizer model; only " + FAMILY_WORDPIECE + " and " + FAMILY_UNIGRAM + " (" + FAMILY_SENTENCEPIECE + ") distillations are supported"); }; @@ -126,7 +129,7 @@ private static Result assembleWordpiece(Path modelDirectory, TokenizerJson token boolean wroteVocabulary = false; if (!Files.exists(vocabularyFile)) { if (tokenizer.orderedVocabulary() == null) { - throw new IllegalArgumentException("tokenizer.json in " + modelDirectory + throw new InvalidFormatException("tokenizer.json in " + modelDirectory + " has no model.vocab dictionary; cannot derive " + ModelFileNames.VOCABULARY); } Files.write(vocabularyFile, tokenizer.orderedVocabulary()); @@ -171,7 +174,7 @@ private static Result assembleSentencePiece(Path modelDirectory) throws IOExcept /** * Loads the assembled directory to verify it, translating a load failure into an assembly - * failure with the same message. + * failure with the same message and the same exception type. * * @param modelDirectory The assembled directory. * @return The loaded model. @@ -180,6 +183,9 @@ private static Result assembleSentencePiece(Path modelDirectory) throws IOExcept private static StaticEmbeddingModel load(Path modelDirectory) throws IOException { try { return StaticEmbeddingModel.load(modelDirectory); + } catch (InvalidFormatException e) { + throw new InvalidFormatException("Assembled directory " + modelDirectory + + " does not load: " + e.getMessage(), e); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Assembled directory " + modelDirectory + " does not load: " + e.getMessage(), e); @@ -220,7 +226,7 @@ private record TokenizerJson(String modelType, List orderedVocabulary, * * @param file The {@code tokenizer.json} file. * @return The parsed fields. - * @throws IllegalArgumentException Thrown if the file is not a well-formed {@code tokenizer.json}. + * @throws InvalidFormatException Thrown if the file is not a well-formed {@code tokenizer.json}. * @throws IOException Thrown if reading the file fails. */ private static TokenizerJson readTokenizerJson(Path file) throws IOException { @@ -263,7 +269,7 @@ private static TokenizerJson readTokenizerJson(Path file) throws IOException { } cursor.requireEnd("Trailing content after the top-level object"); if (modelType == null) { - throw new IllegalArgumentException(file + " has no model.type"); + throw new InvalidFormatException(file + " has no model.type"); } return new TokenizerJson(modelType, orderedVocabulary, lowerCase); } @@ -279,7 +285,7 @@ private record ModelSection(String type, List orderedVocabulary) { * @param cursor The cursor, positioned at the object's opening brace. * @return The parsed type and, for a dictionary vocabulary, the ordered rows. */ - private static ModelSection parseModel(JsonCursor cursor) { + private static ModelSection parseModel(JsonCursor cursor) throws InvalidFormatException { cursor.expect('{'); cursor.skipWhitespace(); String type = null; @@ -319,9 +325,10 @@ private static ModelSection parseModel(JsonCursor cursor) { * * @param cursor The cursor, positioned at the dictionary's opening brace. * @return The tokens in id order. - * @throws IllegalArgumentException Thrown if an id repeats or the ids are not a gapless range. + * @throws InvalidFormatException Thrown if an id repeats or the ids are not a gapless range. */ - private static List parseVocabularyDictionary(JsonCursor cursor) { + private static List parseVocabularyDictionary(JsonCursor cursor) + throws InvalidFormatException { cursor.expect('{'); cursor.skipWhitespace(); final Map tokenById = new LinkedHashMap<>(); @@ -371,7 +378,8 @@ private static List parseVocabularyDictionary(JsonCursor cursor) { * @return The {@code lowercase} flag, or {@code null} when the value is JSON null or the flag is * absent (for example a nested normalizer with no flat flag). */ - private static Boolean parseNormalizerLowercase(JsonCursor cursor) { + private static Boolean parseNormalizerLowercase(JsonCursor cursor) + throws InvalidFormatException { if (cursor.peek() != '{') { cursor.skipValue(); return null; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index 0832e9be92..d377514a3a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -31,6 +31,7 @@ import java.util.Set; import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.util.java.Experimental; /** @@ -94,8 +95,8 @@ private SafetensorsFile(Path file, long dataStart, Map tenso * @param file The file to read. Must not be {@code null} and must exist. * @return The parsed file, with every tensor's metadata resolved and validated against the * file's actual length. - * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing, or the - * file is malformed. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing. + * @throws InvalidFormatException Thrown if the file is malformed. * @throws IOException Thrown if reading the file fails. */ public static SafetensorsFile read(Path file) throws IOException { @@ -108,7 +109,7 @@ public static SafetensorsFile read(Path file) throws IOException { try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { final long fileSize = channel.size(); if (fileSize < HEADER_LENGTH_PREFIX_BYTES) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "File " + file + " is too short to be a safetensors file: " + fileSize + " bytes"); } final ByteBuffer prefix = ByteBuffer.allocate(HEADER_LENGTH_PREFIX_BYTES) @@ -116,11 +117,11 @@ public static SafetensorsFile read(Path file) throws IOException { readFully(channel, prefix, 0, file); final long headerLength = prefix.flip().getLong(); if (headerLength < 0 || headerLength > fileSize - HEADER_LENGTH_PREFIX_BYTES) { - throw new IllegalArgumentException("File " + file + " declares a header length of " + throw new InvalidFormatException("File " + file + " declares a header length of " + headerLength + ", which does not fit in a file of " + fileSize + " bytes"); } if (headerLength > MAX_ARRAY_LENGTH) { - throw new IllegalArgumentException("File " + file + " declares a header length of " + throw new InvalidFormatException("File " + file + " declares a header length of " + headerLength + " bytes, too large to decode as a single JSON string"); } final ByteBuffer headerBytes = ByteBuffer.allocate((int) headerLength); @@ -134,12 +135,12 @@ public static SafetensorsFile read(Path file) throws IOException { for (final TensorInfo tensor : parsed.tensors()) { if (tensor.dataOffsetBegin() < 0 || tensor.dataOffsetEnd() < tensor.dataOffsetBegin() || tensor.dataOffsetEnd() > dataLength) { - throw new IllegalArgumentException("File " + file + " tensor '" + tensor.name() + throw new InvalidFormatException("File " + file + " tensor '" + tensor.name() + "' has a data range [" + tensor.dataOffsetBegin() + ", " + tensor.dataOffsetEnd() + ") that does not fit in the file"); } if (tensorsByName.putIfAbsent(tensor.name(), tensor) != null) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "File " + file + " declares tensor '" + tensor.name() + "' more than once"); } } @@ -181,9 +182,11 @@ public TensorInfo tensorInfo(String name) { * * @param name The tensor's name. Must not be {@code null}. * @return The tensor's elements in row-major (shape outermost-first) order. - * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, not a tensor in - * this file, not a supported float dtype ({@code F32}, {@code F16}, {@code BF16}), or - * larger than a Java array can hold. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null} or not a tensor in + * this file. + * @throws InvalidFormatException Thrown if the tensor is not a supported float dtype + * ({@code F32}, {@code F16}, {@code BF16}), its data range disagrees with its shape, or + * it is larger than a Java array can hold. * @throws IllegalStateException Thrown if the file has been truncated since * {@link #read(Path)} validated the tensor's byte range. * @throws IOException Thrown if reading the file fails. @@ -193,13 +196,13 @@ public float[] readFloats(String name) throws IOException { final int elementBytes = floatElementBytes(info.dtype(), name); final long elementCount = info.elementCount(); if (elementCount < 0 || elementCount > MAX_ARRAY_LENGTH) { - throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount + throw new InvalidFormatException("Tensor '" + name + "' declares " + elementCount + " elements, more than a Java array can hold (" + MAX_ARRAY_LENGTH + "); decoding to a float[] is capped there"); } final long byteLength = info.dataOffsetEnd() - info.dataOffsetBegin(); if (byteLength != elementCount * elementBytes) { - throw new IllegalArgumentException("Tensor '" + name + "' declares " + elementCount + " " + throw new InvalidFormatException("Tensor '" + name + "' declares " + elementCount + " " + info.dtype() + " elements but its data range is " + byteLength + " bytes"); } final float[] values = new float[(int) elementCount]; @@ -232,15 +235,17 @@ public float[] readFloats(String name) throws IOException { * * @param name The tensor's name. Must not be {@code null}. * @return The tensor's elements in row-major (shape outermost-first) order. - * @throws IllegalArgumentException Thrown if {@code name} is {@code null}, not a tensor in - * this file, not declared with dtype {@code F32}, or larger than a Java array can hold. + * @throws IllegalArgumentException Thrown if {@code name} is {@code null} or not a tensor in + * this file. + * @throws InvalidFormatException Thrown if the tensor is not declared with dtype {@code F32}, + * its data range disagrees with its shape, or it is larger than a Java array can hold. * @throws IllegalStateException Thrown if the file has been truncated since {@link #read(Path)}. * @throws IOException Thrown if reading the file fails. */ public float[] readFloat32(String name) throws IOException { final TensorInfo info = tensorInfo(name); if (!DTYPE_F32.equals(info.dtype())) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "Tensor '" + name + "' has dtype " + info.dtype() + ", not " + DTYPE_F32); } return readFloats(name); @@ -281,13 +286,14 @@ private static void decodeInto(ByteBuffer chunk, String dtype, float[] out, int * * @param dtype The tensor dtype. * @param tensorName The tensor's name, for the error message. - * @throws IllegalArgumentException Thrown if {@code dtype} is not a supported float type. + * @throws InvalidFormatException Thrown if {@code dtype} is not a supported float type. */ - private static int floatElementBytes(String dtype, String tensorName) { + private static int floatElementBytes(String dtype, String tensorName) + throws InvalidFormatException { return switch (dtype) { case DTYPE_F32 -> Float.BYTES; case DTYPE_F16, DTYPE_BF16 -> Short.BYTES; - default -> throw new IllegalArgumentException("Tensor '" + tensorName + "' has dtype " + default -> throw new InvalidFormatException("Tensor '" + tensorName + "' has dtype " + dtype + ", not a supported float type (" + DTYPE_F32 + ", " + DTYPE_F16 + ", " + DTYPE_BF16 + ")"); }; @@ -328,16 +334,16 @@ private static void readFully(FileChannel channel, ByteBuffer buffer, long posit * wrong guess cannot silently load the wrong tensor. * * @return The name of the single 2-D float tensor. - * @throws IllegalArgumentException Thrown if the file has zero or more than one 2-D float + * @throws InvalidFormatException Thrown if the file has zero or more than one 2-D float * tensor; the message lists every candidate so the caller can pick explicitly with * {@link #readFloats(String)}. */ - public String singleMatrixTensorName() { + public String singleMatrixTensorName() throws InvalidFormatException { String found = null; for (final TensorInfo info : tensorsByName.values()) { if (isFloatDtype(info.dtype()) && info.shape().length == 2) { if (found != null) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "More than one 2-D float tensor in this file; specify the name explicitly. " + "Candidates: " + tensorsByName.keySet()); } @@ -345,7 +351,7 @@ public String singleMatrixTensorName() { } } if (found == null) { - throw new IllegalArgumentException( + throw new InvalidFormatException( "No 2-D float (F32/F16/BF16) tensor in this file. Available tensors: " + tensorsByName.keySet()); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java index 784a0c754a..8916008966 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsHeaderParser.java @@ -21,6 +21,8 @@ import java.util.List; import java.util.Map; +import opennlp.tools.util.InvalidFormatException; + /** * A cursor parser for the JSON header of a safetensors file: a flat object of tensor name to a * {@code dtype}/{@code shape}/{@code data_offsets} record, plus an optional {@code __metadata__} @@ -44,9 +46,10 @@ private SafetensorsHeaderParser(String text) { * {@code null}. * @return The parse result: the declared tensors, in header order, and the * {@code __metadata__} string map (empty when the header has none). - * @throws IllegalArgumentException Thrown if {@code headerJson} is {@code null} or malformed. + * @throws IllegalArgumentException Thrown if {@code headerJson} is {@code null}. + * @throws InvalidFormatException Thrown if {@code headerJson} is malformed. */ - static Result parse(String headerJson) { + static Result parse(String headerJson) throws InvalidFormatException { if (headerJson == null) { throw new IllegalArgumentException("HeaderJson must not be null"); } @@ -55,7 +58,7 @@ static Result parse(String headerJson) { } /** {@return the parsed header: its tensors in header order and the {@code __metadata__} map} */ - private Result parseTop() { + private Result parseTop() throws InvalidFormatException { final List tensors = new ArrayList<>(); Map metadata = Map.of(); cursor.skipWhitespace(); @@ -95,7 +98,7 @@ private Result parseTop() { * Requires the rest of the header to be whitespace only. Trailing whitespace is legal (writers * space-pad the header to align the data section); other trailing content is a length mismatch. */ - private void requireEnd() { + private void requireEnd() throws InvalidFormatException { cursor.requireEnd("Trailing content after the header object"); } @@ -104,7 +107,7 @@ private void requireEnd() { * * @param name The tensor's name, the key it was declared under. */ - private TensorInfo parseTensorInfo(String name) { + private TensorInfo parseTensorInfo(String name) throws InvalidFormatException { cursor.expect('{'); String dtype = null; int[] shape = null; @@ -151,7 +154,7 @@ private TensorInfo parseTensorInfo(String name) { } /** {@return a JSON object of string values, used for the {@code __metadata__} map} */ - private Map parseStringMap() { + private Map parseStringMap() throws InvalidFormatException { final Map map = new LinkedHashMap<>(); cursor.expect('{'); cursor.skipWhitespace(); @@ -181,9 +184,9 @@ private Map parseStringMap() { /** * {@return a JSON array of non-negative integers as an {@code int[]}} * - * @throws IllegalArgumentException Thrown if any element is outside the {@code int} range. + * @throws InvalidFormatException Thrown if any element is outside the {@code int} range. */ - private int[] parseIntArray() { + private int[] parseIntArray() throws InvalidFormatException { final long[] longs = parseLongArray(); final int[] ints = new int[longs.length]; for (int i = 0; i < longs.length; i++) { @@ -196,7 +199,7 @@ private int[] parseIntArray() { } /** {@return a JSON array of integers as a {@code long[]}} */ - private long[] parseLongArray() { + private long[] parseLongArray() throws InvalidFormatException { cursor.expect('['); cursor.skipWhitespace(); final List values = new ArrayList<>(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index df93b6d7d7..c292241141 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -34,6 +34,7 @@ import opennlp.tools.tokenize.SubwordTokenizer; import opennlp.tools.tokenize.WordpieceEncoder; import opennlp.tools.tokenize.WordpieceTokenizer; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.util.java.Experimental; /** @@ -145,9 +146,10 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, * directory. * @return The loaded model. * @throws IllegalArgumentException Thrown if {@code modelDirectory} is {@code null} or not a - * directory, neither layout's files are present, a configuration file is malformed or - * lacks its field, the accent handling is not representable, or the tokenizer and the - * embedding matrix disagree. + * directory. + * @throws InvalidFormatException Thrown if neither layout's files are present, a required + * file is missing, a configuration file is malformed or lacks its field, the accent + * handling is not representable, or the tokenizer and the embedding matrix disagree. * @throws IOException Thrown if reading a file fails. */ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException { @@ -171,12 +173,12 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG))); } if (Files.isRegularFile(tokenizerJsonFile)) { - throw new IllegalArgumentException("Model directory " + modelDirectory + " has a " + throw new InvalidFormatException("Model directory " + modelDirectory + " has a " + ModelFileNames.TOKENIZER_JSON + " but no trained SentencePiece file (" + String.join(", ", ModelFileNames.SENTENCEPIECE_MODELS) + "); copy the .model file " + "from the model's base tokenizer next to it"); } - throw new IllegalArgumentException("Model directory " + modelDirectory + " has neither a " + throw new InvalidFormatException("Model directory " + modelDirectory + " has neither a " + ModelFileNames.VOCABULARY + " (WordPiece layout) nor a " + ModelFileNames.TOKENIZER_JSON + " with a trained SentencePiece file (SentencePiece layout)"); @@ -202,14 +204,14 @@ private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, final Boolean lowerCase = FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "do_lower_case"); if (lowerCase == null) { - throw new IllegalArgumentException(tokenizerConfigFile + " has no boolean " + throw new InvalidFormatException(tokenizerConfigFile + " has no boolean " + "'do_lower_case' field; use load(vocabularyFile, safetensorsFile, casing, " + "normalization) and choose explicitly"); } final Boolean stripAccents = FlatJsonFields.topLevelBoolean(tokenizerConfigFile, "strip_accents"); if (stripAccents != null && !stripAccents.equals(lowerCase)) { - throw new IllegalArgumentException(tokenizerConfigFile + " sets strip_accents=" + throw new InvalidFormatException(tokenizerConfigFile + " sets strip_accents=" + stripAccents + " against do_lower_case=" + lowerCase + "; the single lower-case " + "switch strips accents exactly when lower-casing, so this model must be loaded " + "with load(vocabularyFile, safetensorsFile, casing, normalization) after choosing " @@ -224,13 +226,13 @@ private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, * * @param configFile The {@code config.json} file. * @return The corresponding {@link Normalization}. - * @throws IllegalArgumentException Thrown if the field is missing or not a boolean. + * @throws InvalidFormatException Thrown if the field is missing or not a boolean. * @throws IOException Thrown if reading the file fails. */ private static Normalization requiredNormalize(Path configFile) throws IOException { final Boolean normalize = FlatJsonFields.topLevelBoolean(configFile, "normalize"); if (normalize == null) { - throw new IllegalArgumentException(configFile + " has no boolean 'normalize' field; " + throw new InvalidFormatException(configFile + " has no boolean 'normalize' field; " + "use the explicit load overloads and choose the normalization deliberately"); } return normalize ? Normalization.L2 : Normalization.NONE; @@ -241,12 +243,13 @@ private static Normalization requiredNormalize(Path configFile) throws IOExcepti * * @param modelDirectory The model directory. * @param name The required file name. - * @throws IllegalArgumentException Thrown if the file is absent. + * @throws InvalidFormatException Thrown if the file is absent. */ - private static Path requiredFile(Path modelDirectory, String name) { + private static Path requiredFile(Path modelDirectory, String name) + throws InvalidFormatException { final Path file = modelDirectory.resolve(name); if (!Files.isRegularFile(file)) { - throw new IllegalArgumentException("Model directory " + modelDirectory + " has no " + throw new InvalidFormatException("Model directory " + modelDirectory + " has no " + name + "; for a different layout, use the explicit load overloads"); } return file; @@ -273,8 +276,11 @@ private static Path requiredFile(Path modelDirectory, String name) { * @param normalization Whether {@link #embed(String)} L2-normalizes its result * ({@link Normalization#L2}) or not ({@link Normalization#NONE}). * @return The loaded model. - * @throws IllegalArgumentException Thrown if an argument is {@code null}, a file is missing - * or malformed, or the vocabulary size and the embedding matrix's row count disagree. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or a file is + * missing. + * @throws InvalidFormatException Thrown if a file is malformed, the vocabulary lacks the + * {@code [UNK]} token, or the vocabulary size and the embedding matrix's row count + * disagree. * @throws IOException Thrown if reading a file fails. */ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFile, @@ -296,7 +302,7 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil final Matrix matrix = readMatrix(vocabulary, safetensorsFile, vocabularyFile.toString()); final int unknownId = vocabulary.id(WordpieceTokenizer.BERT_UNK_TOKEN); if (unknownId < 0) { - throw new IllegalArgumentException("Vocabulary " + vocabularyFile + " has no " + throw new InvalidFormatException("Vocabulary " + vocabularyFile + " has no " + WordpieceTokenizer.BERT_UNK_TOKEN + " token; a WordPiece embedding model needs an " + "unknown token as the fallback for out-of-vocabulary text"); } @@ -372,9 +378,11 @@ private static WordpieceEncoder wordpieceEncoder(EmbeddingVocabulary vocabulary, * @param normalization Whether {@link #embed(String)} L2-normalizes its result * ({@link Normalization#L2}) or not ({@link Normalization#NONE}). * @return The loaded model. - * @throws IllegalArgumentException Thrown if an argument is {@code null}, a file is missing - * or malformed, the vocabulary size and the embedding matrix's row count disagree, or the - * tokenizer emits pieces the vocabulary does not map. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or a file is + * missing. + * @throws InvalidFormatException Thrown if a file is malformed, the vocabulary size and the + * embedding matrix's row count disagree, or the tokenizer emits pieces the vocabulary + * does not map. * @throws IOException Thrown if reading a file fails. */ public static StaticEmbeddingModel loadSentencePiece(Path sentencePieceModelFile, @@ -417,12 +425,13 @@ public static StaticEmbeddingModel loadSentencePiece(Path sentencePieceModelFile * @param vocabulary The matrix row vocabulary. * @param sentencePieceModelFile The tokenizer's source file, for error messages. * @param tokenizerJsonFile The vocabulary's source file, for error messages. - * @throws IllegalArgumentException Thrown if a poolable piece has no matrix row. + * @throws InvalidFormatException Thrown if a poolable piece has no matrix row. */ private static void requireVocabularyCoverage(SentencePieceTokenizer tokenizer, EmbeddingVocabulary vocabulary, Path sentencePieceModelFile, - Path tokenizerJsonFile) { + Path tokenizerJsonFile) + throws InvalidFormatException { int missing = 0; final StringBuilder samples = new StringBuilder(); for (int id = 0; id < tokenizer.vocabularySize(); id++) { @@ -440,7 +449,7 @@ private static void requireVocabularyCoverage(SentencePieceTokenizer tokenizer, } } if (missing > 0) { - throw new IllegalArgumentException(sentencePieceModelFile + " defines " + missing + throw new InvalidFormatException(sentencePieceModelFile + " defines " + missing + " pieces that " + tokenizerJsonFile + " does not map to a matrix row (first: " + samples + "); these files do not belong to the same model"); } @@ -458,7 +467,7 @@ private record Matrix(float[] embeddings, float[] weights, int dimension) { * @param safetensorsFile The safetensors file to read. * @param vocabularySourceName The vocabulary's source, for error messages. * @return The matrix, its optional weights, and its dimension. - * @throws IllegalArgumentException Thrown if the matrix's row count or the weights tensor's + * @throws InvalidFormatException Thrown if the matrix's row count or the weights tensor's * length disagrees with the vocabulary size. * @throws IOException Thrown if reading the file fails. */ @@ -468,7 +477,7 @@ private static Matrix readMatrix(EmbeddingVocabulary vocabulary, Path safetensor final String matrixName = tensors.singleMatrixTensorName(); final TensorInfo matrixInfo = tensors.tensorInfo(matrixName); if (matrixInfo.shape()[0] != vocabulary.size()) { - throw new IllegalArgumentException("Vocabulary " + vocabularySourceName + " has " + throw new InvalidFormatException("Vocabulary " + vocabularySourceName + " has " + vocabulary.size() + " tokens but embedding matrix '" + matrixName + "' in " + safetensorsFile + " has " + matrixInfo.shape()[0] + " rows; these files do not " + "belong to the same model"); @@ -480,7 +489,7 @@ private static Matrix readMatrix(EmbeddingVocabulary vocabulary, Path safetensor if (tensors.tensorNames().contains(WEIGHTS_TENSOR_NAME)) { weights = tensors.readFloats(WEIGHTS_TENSOR_NAME); if (weights.length != vocabulary.size()) { - throw new IllegalArgumentException("Tensor '" + WEIGHTS_TENSOR_NAME + "' in " + throw new InvalidFormatException("Tensor '" + WEIGHTS_TENSOR_NAME + "' in " + safetensorsFile + " has " + weights.length + " elements but the vocabulary has " + vocabulary.size() + " tokens"); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java index dd9dc095c6..a9b5c1584e 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java @@ -28,6 +28,8 @@ import java.util.Set; import java.util.regex.Pattern; +import opennlp.tools.util.InvalidFormatException; + /** * The tokenizer side of a teacher model, distilled the way * Model2Vec distills it. The class reads @@ -104,9 +106,11 @@ private TeacherTokenizer(String json, String inputName, String modelType, * @param tokenizerConfigFile The teacher's {@code tokenizer_config.json}, consulted for the * pad token only; may be {@code null} (no pad token then). * @return The parsed teacher tokenizer. - * @throws IllegalArgumentException Thrown if the files are missing or malformed, the tokenizer - * model is neither WordPiece nor Unigram, the vocabulary ids are not a gapless range, the - * unknown token is missing, or the post-processor is of an unsupported type. + * @throws IllegalArgumentException Thrown if {@code tokenizerJsonFile} is {@code null} or + * missing. + * @throws InvalidFormatException Thrown if a file is malformed, the tokenizer model is + * neither WordPiece nor Unigram, the vocabulary ids are not a gapless range, the unknown + * token is missing, or the post-processor is of an unsupported type. * @throws IOException Thrown if reading a file fails. */ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) @@ -166,11 +170,11 @@ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) } cursor.requireEnd("Trailing content after the top-level object"); if (modelType == null || tokensById == null) { - throw new IllegalArgumentException(tokenizerJsonFile + " has no model with a vocabulary; " + throw new InvalidFormatException(tokenizerJsonFile + " has no model with a vocabulary; " + "it does not look like a teacher's tokenizer.json"); } if (!WORDPIECE.equals(modelType) && !UNIGRAM.equals(modelType)) { - throw new IllegalArgumentException(tokenizerJsonFile + " has a '" + modelType + throw new InvalidFormatException(tokenizerJsonFile + " has a '" + modelType + "' tokenizer model; only " + WORDPIECE + " and " + UNIGRAM + " teachers are supported"); } @@ -180,14 +184,14 @@ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) } if (unkToken == null) { if (unkId == null || unkId < 0 || unkId >= tokensById.size()) { - throw new IllegalArgumentException(tokenizerJsonFile + " does not name an unknown token " + throw new InvalidFormatException(tokenizerJsonFile + " does not name an unknown token " + "(no model.unk_token / model.unk_id); a distilled table needs one"); } unkToken = tokensById.get(unkId.intValue()); } final Integer originalUnkId = idByToken.get(unkToken); if (originalUnkId == null) { - throw new IllegalArgumentException(tokenizerJsonFile + " names the unknown token '" + throw new InvalidFormatException(tokenizerJsonFile + " names the unknown token '" + unkToken + "' but it is not in the vocabulary"); } // The wrapper ids come from the cls/sep pairs of a BertProcessing/RobertaProcessing @@ -232,10 +236,11 @@ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) * @param specialTokenIds The post-processor's name-to-id table. * @param idByToken The vocabulary, token to id. * @param file The source file, for error messages. - * @throws IllegalArgumentException Thrown if a name resolves nowhere. + * @throws InvalidFormatException Thrown if a name resolves nowhere. */ private static int[] resolveNames(List names, Map specialTokenIds, - Map idByToken, Path file) { + Map idByToken, Path file) + throws InvalidFormatException { final int[] ids = new int[names.size()]; for (int i = 0; i < names.size(); i++) { final Long specialId = specialTokenIds.get(names.get(i)); @@ -245,7 +250,7 @@ private static int[] resolveNames(List names, Map specialT } else if (vocabId != null) { ids[i] = vocabId; } else { - throw new IllegalArgumentException(file + " wraps sequences in the special token '" + throw new InvalidFormatException(file + " wraps sequences in the special token '" + names.get(i) + "' but neither the post-processor nor the vocabulary defines it"); } } @@ -379,7 +384,8 @@ void writeCleaned(Path file) throws IOException { * @param newIdByOriginal The original-to-new id map. */ private void rewriteModel(JsonCursor cursor, StringBuilder out, - Map newIdByOriginal) { + Map newIdByOriginal) + throws InvalidFormatException { cursor.expect('{'); out.append('{'); cursor.skipWhitespace(); @@ -431,7 +437,8 @@ private void rewriteModel(JsonCursor cursor, StringBuilder out, * @param cursor The cursor, positioned at the vocabulary's opening character. * @param newIdByOriginal The original-to-new id map. */ - private String rewrittenVocab(JsonCursor cursor, Map newIdByOriginal) { + private String rewrittenVocab(JsonCursor cursor, Map newIdByOriginal) + throws InvalidFormatException { final StringBuilder out = new StringBuilder(); if (cursor.peek() == '{') { cursor.consume(); @@ -586,7 +593,7 @@ private static String quoted(String content) { * * @param cursor The cursor, positioned at the value. */ - private String copyRawValue(JsonCursor cursor) { + private String copyRawValue(JsonCursor cursor) throws InvalidFormatException { final int start = cursor.position(); cursor.skipValue(); return json.substring(start, cursor.position()); @@ -603,7 +610,7 @@ private record ModelSection(String type, List tokensById, String unkToke * @param cursor The cursor, positioned at the object's opening brace. * @return The parsed section. */ - private static ModelSection parseModel(JsonCursor cursor) { + private static ModelSection parseModel(JsonCursor cursor) throws InvalidFormatException { cursor.expect('{'); cursor.skipWhitespace(); String type = null; @@ -649,7 +656,7 @@ private static ModelSection parseModel(JsonCursor cursor) { * * @param cursor The cursor, positioned at the vocabulary's opening character. */ - private static List parseVocab(JsonCursor cursor) { + private static List parseVocab(JsonCursor cursor) throws InvalidFormatException { if (cursor.peek() == '{') { cursor.consume(); cursor.skipWhitespace(); @@ -725,7 +732,8 @@ private static List parseVocab(JsonCursor cursor) { * * @param cursor The cursor, positioned at the list's opening bracket. */ - private static Set parseAddedTokenContents(JsonCursor cursor) { + private static Set parseAddedTokenContents(JsonCursor cursor) + throws InvalidFormatException { cursor.expect('['); cursor.skipWhitespace(); final Set contents = new HashSet<>(); @@ -792,9 +800,10 @@ private record PostProcessor(List bosNames, List eosNames, Long * * @param cursor The cursor, positioned at the value. * @return The parsed post-processor. - * @throws IllegalArgumentException Thrown if the type is not one of the supported forms. + * @throws InvalidFormatException Thrown if the type is not one of the supported forms. */ - private static PostProcessor parsePostProcessor(JsonCursor cursor) { + private static PostProcessor parsePostProcessor(JsonCursor cursor) + throws InvalidFormatException { if (cursor.consumeLiteral("null")) { return new PostProcessor(List.of(), List.of(), null, null, Map.of()); } @@ -847,7 +856,7 @@ private static PostProcessor parsePostProcessor(JsonCursor cursor) { new PostProcessor(bosNames, eosNames, null, null, specialTokenIds); case "BertProcessing", "RobertaProcessing" -> new PostProcessor(List.of(), List.of(), clsId, sepId, specialTokenIds); - default -> throw new IllegalArgumentException("The post_processor type '" + type + default -> throw new InvalidFormatException("The post_processor type '" + type + "' is not supported; expected TemplateProcessing, BertProcessing, or " + "RobertaProcessing"); }; @@ -861,7 +870,8 @@ private static PostProcessor parsePostProcessor(JsonCursor cursor) { * * @param cursor The cursor, positioned at the template value. */ - private static List> parseTemplate(JsonCursor cursor) { + private static List> parseTemplate(JsonCursor cursor) + throws InvalidFormatException { final List bos = new ArrayList<>(1); final List eos = new ArrayList<>(1); if (cursor.peek() == '"') { @@ -950,7 +960,8 @@ private static List> parseTemplate(JsonCursor cursor) { * * @param cursor The cursor, positioned at the table's opening brace. */ - private static Map parseSpecialTokenIds(JsonCursor cursor) { + private static Map parseSpecialTokenIds(JsonCursor cursor) + throws InvalidFormatException { cursor.expect('{'); cursor.skipWhitespace(); final Map ids = new HashMap<>(); @@ -1021,7 +1032,7 @@ private static Map parseSpecialTokenIds(JsonCursor cursor) { * * @param cursor The cursor, positioned at the pair's opening bracket. */ - private static Long parseTokenIdPair(JsonCursor cursor) { + private static Long parseTokenIdPair(JsonCursor cursor) throws InvalidFormatException { cursor.expect('['); cursor.skipWhitespace(); cursor.parseString(); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java index 6f5d96d5da..3d918ee371 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TokenizerJsonVocab.java @@ -23,6 +23,8 @@ import java.util.Comparator; import java.util.List; +import opennlp.tools.util.InvalidFormatException; + /** * Reads the row order of a static embedding matrix out of a {@code tokenizer.json} file with a * Unigram model: the {@code model.vocab} list holds {@code [piece, score]} pairs whose index is @@ -53,9 +55,10 @@ private record AddedToken(long id, String content) { * * @param file The {@code tokenizer.json} file. Must not be {@code null} and must exist. * @return The pieces; the index is the matrix row. - * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing, the - * file is not a well-formed {@code tokenizer.json}, its model is not Unigram, or an added - * token's id neither matches an existing row nor appends as the next one. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing. + * @throws InvalidFormatException Thrown if the file is not a well-formed + * {@code tokenizer.json}, its model is not Unigram, or an added token's id neither + * matches an existing row nor appends as the next one. * @throws IOException Thrown if reading the file fails. */ static List rows(Path file) throws IOException { @@ -119,12 +122,12 @@ static List rows(Path file) throws IOException { cursor.requireEnd("Trailing content after the top-level object"); if (modelType != null && !"Unigram".equals(modelType)) { - throw new IllegalArgumentException(file + " has a '" + modelType + "' tokenizer model; " + throw new InvalidFormatException(file + " has a '" + modelType + "' tokenizer model; " + "only the Unigram list layout maps pieces to matrix rows here. For a WordPiece " + "model, load from its vocab.txt instead"); } if (vocab == null) { - throw new IllegalArgumentException(file + " has no model.vocab list; it does not name " + throw new InvalidFormatException(file + " has no model.vocab list; it does not name " + "the matrix rows"); } return overlayAddedTokens(vocab, addedTokens, file); @@ -141,7 +144,7 @@ private record ParsedModel(String type, List vocab) { * @param cursor The cursor, positioned at the object's opening brace. * @return The parsed type and vocabulary; either may be absent ({@code null}). */ - private static ParsedModel parseModel(JsonCursor cursor) { + private static ParsedModel parseModel(JsonCursor cursor) throws InvalidFormatException { cursor.expect('{'); cursor.skipWhitespace(); String type = null; @@ -193,7 +196,8 @@ private static ParsedModel parseModel(JsonCursor cursor) { * @param cursor The cursor, positioned at the list's opening bracket. * @return The pieces in list order. */ - private static List parseVocabList(JsonCursor cursor) { + private static List parseVocabList(JsonCursor cursor) + throws InvalidFormatException { cursor.expect('['); cursor.skipWhitespace(); final List pieces = new ArrayList<>(); @@ -231,7 +235,8 @@ private static List parseVocabList(JsonCursor cursor) { * @param cursor The cursor, positioned at the list's opening bracket. * @return The added tokens in list order. */ - private static List parseAddedTokens(JsonCursor cursor) { + private static List parseAddedTokens(JsonCursor cursor) + throws InvalidFormatException { cursor.expect('['); cursor.skipWhitespace(); final List tokens = new ArrayList<>(); @@ -260,7 +265,8 @@ private static List parseAddedTokens(JsonCursor cursor) { * @param cursor The cursor, positioned at the object's opening brace. * @return The parsed entry. */ - private static AddedToken parseAddedToken(JsonCursor cursor) { + private static AddedToken parseAddedToken(JsonCursor cursor) + throws InvalidFormatException { cursor.expect('{'); cursor.skipWhitespace(); Long id = null; @@ -317,9 +323,12 @@ private static AddedToken parseAddedToken(JsonCursor cursor) { * @param addedTokens The added tokens to overlay. * @param file The source file, for error messages. * @return The vocabulary with the added tokens applied. + * @throws InvalidFormatException Thrown if an added token contradicts the vocabulary or + * leaves a gap in the id space. */ private static List overlayAddedTokens(List vocab, - List addedTokens, Path file) { + List addedTokens, Path file) + throws InvalidFormatException { final List byId = new ArrayList<>(addedTokens); byId.sort(Comparator.comparingLong(AddedToken::id)); for (final AddedToken token : byId) { @@ -328,12 +337,12 @@ private static List overlayAddedTokens(List vocab, } else if (token.id() < vocab.size()) { final String existing = vocab.get((int) token.id()); if (!existing.equals(token.content())) { - throw new IllegalArgumentException(file + " declares added token '" + token.content() + throw new InvalidFormatException(file + " declares added token '" + token.content() + "' at id " + token.id() + " but model.vocab holds '" + existing + "' there; the file contradicts itself"); } } else { - throw new IllegalArgumentException(file + " declares added token '" + token.content() + throw new InvalidFormatException(file + " declares added token '" + token.content() + "' at id " + token.id() + " but the vocabulary only has " + vocab.size() + " rows; the id space has a gap"); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java index d1a97bdda4..c08fdbd225 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java @@ -22,6 +22,7 @@ import opennlp.embeddings.ModelAssembler; import opennlp.tools.cmdline.BasicCmdLineTool; import opennlp.tools.cmdline.TerminateToolException; +import opennlp.tools.util.InvalidFormatException; /** * Completes a distilled embedding model directory so {@code StaticEmbeddingModel.load} can open it, @@ -60,7 +61,7 @@ public void run(String[] args) { final ModelAssembler.Result result; try { result = ModelAssembler.assemble(modelDir.toPath()); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidFormatException e) { throw new TerminateToolException(1, e.getMessage(), e); } catch (IOException e) { throw new TerminateToolException(-1, diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java index 5d0c7a74ff..36f0a71b27 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java @@ -22,6 +22,7 @@ import opennlp.embeddings.ModelDistiller; import opennlp.tools.cmdline.BasicCmdLineTool; import opennlp.tools.cmdline.TerminateToolException; +import opennlp.tools.util.InvalidFormatException; /** * Distills a sentence-transformer teacher into a static embedding model directory, the @@ -61,7 +62,7 @@ public void run(String[] args) { try { result = ModelDistiller.distill(params.getTeacher(), Path.of(params.getOut()), params.getPcaDims(), listener); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | InvalidFormatException e) { throw new TerminateToolException(1, e.getMessage(), e); } catch (IOException e) { throw new TerminateToolException(-1, diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingVocabularyTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingVocabularyTest.java index 92ae7ee68f..d11a6297c7 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingVocabularyTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingVocabularyTest.java @@ -24,6 +24,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -35,7 +37,7 @@ class EmbeddingVocabularyTest { @Test - void testLineNumberIsTheTokenId() { + void testLineNumberIsTheTokenId() throws InvalidFormatException { final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromLines(List.of("[CLS]", "[SEP]", "hello", "world"), "test"); assertEquals(4, vocabulary.size()); @@ -46,7 +48,7 @@ void testLineNumberIsTheTokenId() { } @Test - void testUnknownTokenIdIsTheSentinel() { + void testUnknownTokenIdIsTheSentinel() throws InvalidFormatException { final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromLines(List.of("hello"), "test"); assertEquals(-1, vocabulary.id("missing")); @@ -55,14 +57,14 @@ void testUnknownTokenIdIsTheSentinel() { @Test void testDuplicateTokenFailsLoudlyNamingBothLines() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> EmbeddingVocabulary.fromLines(List.of("hello", "world", "hello"), "test")); assertTrue(e.getMessage().contains("hello"), e.getMessage()); assertTrue(e.getMessage().contains("0") && e.getMessage().contains("2"), e.getMessage()); } @Test - void testReverseLookupEnforcesBounds() { + void testReverseLookupEnforcesBounds() throws InvalidFormatException { final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromLines(List.of("hello"), "test"); assertEquals("hello", vocabulary.token(0)); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java index d4f728ba06..6ceb7a179d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/FlatJsonFieldsTest.java @@ -23,6 +23,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -79,7 +81,7 @@ void testToleratesAnEmptyObjectAndTrailingWhitespace(@TempDir Path dir) throws I void testRejectsANonBooleanValue(@TempDir Path dir) throws IOException { final Path file = write(dir, "{\"normalize\":\"yes\"}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> FlatJsonFields.topLevelBoolean(file, "normalize")); assertTrue(e.getMessage().contains("must be a boolean")); } @@ -88,7 +90,7 @@ void testRejectsANonBooleanValue(@TempDir Path dir) throws IOException { void testRejectsADuplicateField(@TempDir Path dir) throws IOException { final Path file = write(dir, "{\"normalize\":true,\"normalize\":false}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> FlatJsonFields.topLevelBoolean(file, "normalize")); assertTrue(e.getMessage().contains("more than once")); } @@ -97,7 +99,7 @@ void testRejectsADuplicateField(@TempDir Path dir) throws IOException { void testRejectsMalformedJsonWithTheFileNameInTheMessage(@TempDir Path dir) throws IOException { final Path file = write(dir, "{\"normalize\" true}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> FlatJsonFields.topLevelBoolean(file, "normalize")); assertTrue(e.getMessage().contains("config.json")); } @@ -106,7 +108,7 @@ void testRejectsMalformedJsonWithTheFileNameInTheMessage(@TempDir Path dir) thro void testRejectsTrailingGarbage(@TempDir Path dir) throws IOException { final Path file = write(dir, "{} x"); - assertThrows(IllegalArgumentException.class, + assertThrows(InvalidFormatException.class, () -> FlatJsonFields.topLevelBoolean(file, "normalize")); } @@ -143,7 +145,7 @@ void testNestedOccurrencesOfAStringNameDoNotMatch(@TempDir Path dir) throws IOEx void testRejectsANonStringValue(@TempDir Path dir) throws IOException { final Path file = write(dir, "{\"pad_token\":true}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> FlatJsonFields.topLevelString(file, "pad_token")); assertTrue(e.getMessage().contains("must be a string")); } @@ -152,7 +154,7 @@ void testRejectsANonStringValue(@TempDir Path dir) throws IOException { void testRejectsADuplicateStringField(@TempDir Path dir) throws IOException { final Path file = write(dir, "{\"pad_token\":\"a\",\"pad_token\":\"b\"}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> FlatJsonFields.topLevelString(file, "pad_token")); assertTrue(e.getMessage().contains("more than once")); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java index 07bcf31542..6101d3ecca 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsFileTest.java @@ -31,6 +31,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -150,7 +152,7 @@ void testSingleMatrixTensorNameRejectsAmbiguity(@TempDir Path dir) throws IOExce final SafetensorsFile parsed = SafetensorsFile.read(file); - assertThrows(IllegalArgumentException.class, parsed::singleMatrixTensorName); + assertThrows(InvalidFormatException.class, parsed::singleMatrixTensorName); } @Test @@ -160,7 +162,7 @@ void testSingleMatrixTensorNameRejectsNoCandidate(@TempDir Path dir) throws IOEx final SafetensorsFile parsed = SafetensorsFile.read(file); - assertThrows(IllegalArgumentException.class, parsed::singleMatrixTensorName); + assertThrows(InvalidFormatException.class, parsed::singleMatrixTensorName); } @Test @@ -171,8 +173,8 @@ void testReadFloat32RejectsWrongDtype(@TempDir Path dir) throws IOException { final SafetensorsFile parsed = SafetensorsFile.read(file); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("ids")); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> parsed.readFloat32("ids")); assertTrue(e.getMessage().contains("I64")); } @@ -199,7 +201,7 @@ void testRejectsFileShorterThanTheLengthPrefix(@TempDir Path dir) throws IOExcep final Path file = dir.resolve("truncated.safetensors"); Files.write(file, new byte[] {1, 2, 3}); - assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + assertThrows(InvalidFormatException.class, () -> SafetensorsFile.read(file)); } @Test @@ -209,7 +211,7 @@ void testRejectsHeaderLengthLargerThanTheFile(@TempDir Path dir) throws IOExcept .putLong(1000L).array(); Files.write(file, prefix); - assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + assertThrows(InvalidFormatException.class, () -> SafetensorsFile.read(file)); } @Test @@ -220,8 +222,8 @@ void testRejectsDuplicateTensorName(@TempDir Path dir) throws IOException { + "\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[] {1, 2, 3, 4}); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> SafetensorsFile.read(file)); assertTrue(e.getMessage().contains("more than once")); } @@ -230,7 +232,7 @@ void testRejectsTensorMissingRequiredField(@TempDir Path dir) throws IOException final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]}}"; final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[0]); - assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + assertThrows(InvalidFormatException.class, () -> SafetensorsFile.read(file)); } @Test @@ -238,7 +240,7 @@ void testRejectsDataOffsetsOutOfRange(@TempDir Path dir) throws IOException { final String header = singleTensorHeader("w", "F32", "[1]", 0, 999); final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[] {1, 2, 3, 4}); - assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + assertThrows(InvalidFormatException.class, () -> SafetensorsFile.read(file)); } @Test @@ -246,7 +248,7 @@ void testRejectsUnterminatedString(@TempDir Path dir) throws IOException { final String header = "{\"w\":{\"dtype\":\"F32"; final Path file = writeFile(dir, MODEL_FILE_NAME, header, new byte[0]); - assertThrows(IllegalArgumentException.class, () -> SafetensorsFile.read(file)); + assertThrows(InvalidFormatException.class, () -> SafetensorsFile.read(file)); } @Test @@ -259,8 +261,8 @@ void testRejectsTensorLargerThanAJavaArray(@TempDir Path dir) throws IOException final SafetensorsFile parsed = SafetensorsFile.read(file); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("w")); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> parsed.readFloat32("w")); assertTrue(e.getMessage().contains("more than a Java array can hold")); } @@ -288,8 +290,8 @@ void testReadFloat32RejectsElementCountByteRangeMismatch(@TempDir Path dir) thro final Path file = writeFile(dir, MODEL_FILE_NAME, header, data); final SafetensorsFile parsed = SafetensorsFile.read(file); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("w")); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> parsed.readFloat32("w")); assertTrue(e.getMessage().contains("2 F32 elements"), e.getMessage()); } @@ -353,6 +355,6 @@ void testReadFloat32StrictlyRejectsF16(@TempDir Path dir) throws IOException { final SafetensorsFile parsed = SafetensorsFile.read(file); // readFloats accepts it; the strict readFloat32 must not. assertArrayEquals(new float[] {1f, 2f}, parsed.readFloats("w"), 1e-3f); - assertThrows(IllegalArgumentException.class, () -> parsed.readFloat32("w")); + assertThrows(InvalidFormatException.class, () -> parsed.readFloat32("w")); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java index 7c61709929..95219488b3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SafetensorsHeaderParserTest.java @@ -22,6 +22,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -35,7 +37,7 @@ class SafetensorsHeaderParserTest { @Test - void testParsesTensorsInHeaderOrder() { + void testParsesTensorsInHeaderOrder() throws InvalidFormatException { final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( "{\"beta\":{\"dtype\":\"F32\",\"shape\":[2,3],\"data_offsets\":[0,24]}," + "\"alpha\":{\"dtype\":\"I64\",\"shape\":[],\"data_offsets\":[24,32]}}"); @@ -55,7 +57,7 @@ void testParsesTensorsInHeaderOrder() { } @Test - void testParsesAnEmptyHeader() { + void testParsesAnEmptyHeader() throws InvalidFormatException { final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse("{}"); assertTrue(result.tensors().isEmpty()); @@ -63,7 +65,7 @@ void testParsesAnEmptyHeader() { } @Test - void testParsesAMetadataOnlyHeader() { + void testParsesAMetadataOnlyHeader() throws InvalidFormatException { final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse("{\"__metadata__\":{\"format\":\"pt\"}}"); @@ -72,7 +74,7 @@ void testParsesAMetadataOnlyHeader() { } @Test - void testDecodesEveryEscapeSequence() { + void testDecodesEveryEscapeSequence() throws InvalidFormatException { final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( "{\"__metadata__\":{\"note\":\"\\\"\\\\\\/\\b\\f\\n\\r\\t\\u0041\"}}"); @@ -80,7 +82,7 @@ void testDecodesEveryEscapeSequence() { } @Test - void testSkipsUnknownFieldsOfEveryValueType() { + void testSkipsUnknownFieldsOfEveryValueType() throws InvalidFormatException { // Fields safetensors may add over time must not break the reader: nested objects, arrays, // floating-point numbers, booleans, null, and strings are all skipped structurally. final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( @@ -91,7 +93,7 @@ void testSkipsUnknownFieldsOfEveryValueType() { } @Test - void testToleratesTrailingWhitespacePadding() { + void testToleratesTrailingWhitespacePadding() throws InvalidFormatException { // Writers space-pad the header so the data section starts aligned; padding is part of the // declared header length and must parse cleanly. final SafetensorsHeaderParser.Result result = SafetensorsHeaderParser.parse( @@ -102,7 +104,7 @@ void testToleratesTrailingWhitespacePadding() { @Test void testRejectsTrailingGarbage() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> SafetensorsHeaderParser.parse("{} x")); assertTrue(e.getMessage().contains("Trailing content")); } @@ -150,7 +152,7 @@ void testRejectsNull() { "{\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}" }) void testRejectsMalformedHeaders(String header) { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> SafetensorsHeaderParser.parse(header)); assertTrue(e.getMessage().contains("Malformed safetensors header at offset"), () -> "Message should carry the offset, got: " + e.getMessage()); @@ -161,7 +163,7 @@ void testSignedUnicodeEscapeFailsLoudly() { // Integer.parseInt would accept "-0FF" and decode the wrong character; the parser must not. final String header = "{\"__metadata__\":{\"note\":\"a\\u-0FFb\"}," + "\"w\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[0,4]}}"; - assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(header)); + assertThrows(InvalidFormatException.class, () -> SafetensorsHeaderParser.parse(header)); } @Test @@ -169,7 +171,7 @@ void testMalformedNumberInSkippedFieldFailsLoudly() { // Skipped unknown fields still hold values to the JSON grammar; "1e++--..5" is not a number. final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]," + "\"data_offsets\":[0,4],\"unknown\":1e++--..5}}"; - assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(header)); + assertThrows(InvalidFormatException.class, () -> SafetensorsHeaderParser.parse(header)); } @Test @@ -177,11 +179,11 @@ void testLoneMinusInSkippedFieldFailsLoudly() { // A bare "-" is not a JSON number; the skip path must reject it rather than treating it as one. final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]," + "\"data_offsets\":[0,4],\"unknown\":-}}"; - assertThrows(IllegalArgumentException.class, () -> SafetensorsHeaderParser.parse(header)); + assertThrows(InvalidFormatException.class, () -> SafetensorsHeaderParser.parse(header)); } @Test - void testWellFormedNumbersInSkippedFieldsAreAccepted() { + void testWellFormedNumbersInSkippedFieldsAreAccepted() throws InvalidFormatException { final String header = "{\"w\":{\"dtype\":\"F32\",\"shape\":[1]," + "\"data_offsets\":[0,4],\"a\":-1.5e+10,\"b\":0.25,\"c\":3}}"; assertEquals(1, SafetensorsHeaderParser.parse(header).tensors().size()); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java index ba2289c085..298e1a771b 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java @@ -30,6 +30,7 @@ import opennlp.embeddings.StaticEmbeddingModel.Normalization; import opennlp.subword.sentencepiece.SentencePieceTokenizer; import opennlp.tools.tokenize.SubwordPiece; +import opennlp.tools.util.InvalidFormatException; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -250,8 +251,8 @@ void testLoadRejectsAVocabularyMissingAPoolablePiece(@TempDir Path dir) throws I SafetensorsTestFiles.write(dir.resolve("model.safetensors"), SafetensorsTestFiles.matrix("embeddings", matrix)); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> loadFromDirectory(dir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> loadFromDirectory(dir)); assertTrue(e.getMessage().contains("do not belong"), e.getMessage()); } @@ -262,8 +263,8 @@ void testLoadRejectsARowCountMismatch(@TempDir Path dir) throws IOException { SafetensorsTestFiles.write(dir.resolve("model.safetensors"), SafetensorsTestFiles.matrix("embeddings", matrix)); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> loadFromDirectory(dir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> loadFromDirectory(dir)); assertTrue(e.getMessage().contains("rows"), e.getMessage()); } @@ -272,8 +273,8 @@ void testDirectoryLoadNamesTheMissingSentencePieceModel(@TempDir Path dir) throw writeModelDirectory(dir, true); Files.delete(dir.resolve("sentencepiece.bpe.model")); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); assertTrue(e.getMessage().contains("copy the .model"), e.getMessage()); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index eed96e783f..9ebfbfc187 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -27,6 +27,7 @@ import opennlp.embeddings.StaticEmbeddingModel.Casing; import opennlp.embeddings.StaticEmbeddingModel.Normalization; import opennlp.tools.embeddings.TextEmbedder; +import opennlp.tools.util.InvalidFormatException; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -145,7 +146,7 @@ void testRejectsAWordPieceVocabularyWithoutUnknownToken(@TempDir Path dir) throw final Path tensors = dir.resolve("model.safetensors"); SafetensorsTestFiles.write(tensors, SafetensorsTestFiles.matrix("embeddings", rows)); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(vocab, tensors, Casing.UNCASED, Normalization.NONE)); assertTrue(e.getMessage().contains("[UNK]"), e.getMessage()); } @@ -270,7 +271,9 @@ void testLoadRejectsVocabularySizeMismatch(@TempDir Path dir) throws IOException final Path shortVocab = dir.resolve("short-vocab.txt"); Files.write(shortVocab, List.of("[CLS]", "[SEP]", "[UNK]")); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + // Malformed model content (files that disagree) is a checked InvalidFormatException, not + // an IllegalArgumentException; the latter is reserved for caller argument errors. + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(shortVocab, writeSafetensors(dir, false), Casing.UNCASED, Normalization.NONE)); assertTrue(e.getMessage().contains("rows")); @@ -284,7 +287,7 @@ void testLoadRejectsWeightsSizeMismatch(@TempDir Path dir) throws IOException { SafetensorsTestFiles.matrix("embeddings", ROWS), SafetensorsTestFiles.vector("weights", new float[] {1f})); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(writeVocab(dir), file, Casing.UNCASED, Normalization.NONE)); assertTrue(e.getMessage().contains("weights")); } @@ -344,8 +347,8 @@ void testDirectoryLoadNamesTheMissingFile(@TempDir Path dir) throws IOException writeSafetensors(dir, false); // no config.json, no tokenizer_config.json - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); assertTrue(e.getMessage().contains("config.json")); assertTrue(e.getMessage().contains("explicit load overloads")); } @@ -357,8 +360,8 @@ void testDirectoryLoadRejectsAConfigWithoutNormalize(@TempDir Path dir) throws I writeConfigs(dir, "false", "true"); Files.writeString(dir.resolve("config.json"), "{\"model_type\":\"model2vec\"}"); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); assertTrue(e.getMessage().contains("normalize")); } @@ -370,8 +373,8 @@ void testDirectoryLoadRejectsContradictoryStripAccents(@TempDir Path dir) throws Files.writeString(dir.resolve("tokenizer_config.json"), "{\"do_lower_case\":true,\"strip_accents\":false}"); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(dir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); assertTrue(e.getMessage().contains("strip_accents")); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java index ef00de07fe..df1bc9c615 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java @@ -27,6 +27,8 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -185,7 +187,7 @@ void testRejectsAnUnsupportedModelType(@TempDir Path dir) throws IOException { final Path tokenizerJson = write(dir, "tokenizer.json", "{\"model\":{\"type\":\"BPE\",\"vocab\":{\"a\":0}}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TeacherTokenizer.read(tokenizerJson, null)); assertTrue(e.getMessage().contains("BPE"), e.getMessage()); } @@ -232,7 +234,7 @@ void testRejectsATeacherItCannotDistill(String reason, String teacherJson, Strin @TempDir Path dir) throws IOException { final Path tokenizerJson = write(dir, "tokenizer.json", teacherJson); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TeacherTokenizer.read(tokenizerJson, null), reason); assertTrue(e.getMessage().contains(messagePart), "a teacher with " + reason + " reported: " + e.getMessage()); @@ -333,7 +335,7 @@ void testRejectsATemplateSpecialTokenThatResolvesNowhere(@TempDir Path dir) thro + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + "\"vocab\":{\"[UNK]\":0,\"hello\":1}}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TeacherTokenizer.read(tokenizerJson, null)); assertTrue(e.getMessage().contains("[BOS]"), e.getMessage()); } @@ -343,7 +345,7 @@ void testRejectsAVocabularyIdUsedTwice(@TempDir Path dir) throws IOException { final Path tokenizerJson = write(dir, "tokenizer.json", "{\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"a\",\"vocab\":{\"a\":0,\"b\":0}}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TeacherTokenizer.read(tokenizerJson, null)); assertTrue(e.getMessage().contains("assigned more than once"), e.getMessage()); } @@ -353,7 +355,7 @@ void testAnExplicitlyNullUnkIdCountsAsNoUnknownToken(@TempDir Path dir) throws I final Path tokenizerJson = write(dir, "tokenizer.json", "{\"model\":{\"type\":\"Unigram\",\"unk_id\":null,\"vocab\":[[\"a\",0.0]]}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TeacherTokenizer.read(tokenizerJson, null)); assertTrue(e.getMessage().contains("does not name an unknown token"), e.getMessage()); } @@ -363,7 +365,7 @@ void testAnExplicitlyNullUnkIdCountsAsNoUnknownToken(@TempDir Path dir) throws I void testRejectsAnEmptyTokenizerJson(String content, @TempDir Path dir) throws IOException { final Path tokenizerJson = write(dir, "tokenizer.json", content); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TeacherTokenizer.read(tokenizerJson, null)); assertTrue(e.getMessage().contains("Unexpected end of input"), e.getMessage()); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java index c389e33e1c..937a625630 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TokenizerJsonVocabTest.java @@ -24,6 +24,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -76,7 +78,7 @@ void testAddedTokenAtAnExistingRowMustAgree() throws IOException { final Path contradicting = write("{\"added_tokens\":[{\"id\":0,\"content\":\"\"}]," + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"\",0.0],[\"a\",-1.0]]}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(contradicting)); assertTrue(e.getMessage().contains("contradicts"), e.getMessage()); } @@ -86,7 +88,7 @@ void testAddedTokenBeyondTheNextRowIsAGap() throws IOException { final Path file = write("{\"added_tokens\":[{\"id\":5,\"content\":\"\"}]," + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]]}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(file)); assertTrue(e.getMessage().contains("gap"), e.getMessage()); } @@ -115,7 +117,7 @@ void testSkipsUnrelatedSectionsAndDecodesEscapes() throws IOException { void testRejectsANonUnigramModel() throws IOException { final Path file = write("{\"model\":{\"type\":\"BPE\",\"vocab\":[[\"a\",0.0]]}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(file)); assertTrue(e.getMessage().contains("BPE"), e.getMessage()); } @@ -126,7 +128,7 @@ void testRejectsAnObjectShapedVocab() throws IOException { // are not list positions, so it must be refused rather than misread. final Path file = write("{\"model\":{\"type\":\"Unigram\",\"vocab\":{\"a\":0,\"b\":1}}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(file)); assertTrue(e.getMessage().contains("object"), e.getMessage()); } @@ -134,11 +136,11 @@ void testRejectsAnObjectShapedVocab() throws IOException { @Test void testRejectsAMissingVocab() throws IOException { final Path noModel = write("{\"version\":\"1.0\"}"); - assertTrue(assertThrows(IllegalArgumentException.class, + assertTrue(assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(noModel)).getMessage().contains("model.vocab")); final Path noVocab = write("{\"model\":{\"type\":\"Unigram\"}}"); - assertTrue(assertThrows(IllegalArgumentException.class, + assertTrue(assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(noVocab)).getMessage().contains("model.vocab")); } @@ -147,7 +149,7 @@ void testRejectsAnAddedTokenWithoutIdOrContent() throws IOException { final Path file = write("{\"added_tokens\":[{\"content\":\"\"}]," + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]]}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(file)); assertTrue(e.getMessage().contains("id"), e.getMessage()); } @@ -157,7 +159,7 @@ void testRejectsDuplicateTopLevelSections() throws IOException { final Path file = write("{\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]]}," + "\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"b\",0.0]]}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(file)); assertTrue(e.getMessage().contains("more than once"), e.getMessage()); } @@ -166,7 +168,7 @@ void testRejectsDuplicateTopLevelSections() throws IOException { void testRejectsMalformedJson() throws IOException { final Path file = write("{\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"a\",0.0]"); - assertThrows(IllegalArgumentException.class, () -> TokenizerJsonVocab.rows(file)); + assertThrows(InvalidFormatException.class, () -> TokenizerJsonVocab.rows(file)); } @Test @@ -174,7 +176,7 @@ void testVocabularyEntryPointRejectsDuplicatePieces() throws IOException { final Path file = write("{\"model\":{\"type\":\"Unigram\"," + "\"vocab\":[[\"a\",0.0],[\"a\",-1.0]]}}"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> EmbeddingVocabulary.fromTokenizerJson(file)); assertTrue(e.getMessage().contains("more than once"), e.getMessage()); } From 7212e40d039bbe10e1a8264d53285e92795888c5 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:43:16 -0400 Subject: [PATCH 61/82] OPENNLP-1877: Demote test-only SafetensorsFile accessors to package-private readFloat32(String) and metadata() have no main-source callers; only the module's own tests use them. Shrink the experimental public surface. --- .../src/main/java/opennlp/embeddings/SafetensorsFile.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java index d377514a3a..1d166e1002 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/SafetensorsFile.java @@ -242,7 +242,7 @@ public float[] readFloats(String name) throws IOException { * @throws IllegalStateException Thrown if the file has been truncated since {@link #read(Path)}. * @throws IOException Thrown if reading the file fails. */ - public float[] readFloat32(String name) throws IOException { + float[] readFloat32(String name) throws IOException { final TensorInfo info = tensorInfo(name); if (!DTYPE_F32.equals(info.dtype())) { throw new InvalidFormatException( @@ -359,7 +359,7 @@ public String singleMatrixTensorName() throws InvalidFormatException { } /** {@return the file's {@code __metadata__} string map, empty when the header has none} */ - public Map metadata() { + Map metadata() { return metadata; } From ddb1992bba02b27a0c5848e2b30a769555ff7a88 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:43:16 -0400 Subject: [PATCH 62/82] OPENNLP-1877: Document the DistillModel and AssembleModel tools in the manual Add a Command Line Tools section to the embeddings chapter covering the bin/embeddings launcher and both tools: purpose, invocation shape, and the load-based verification each run ends with, in the style of the manual's other tool sections. --- opennlp-docs/src/docbkx/embeddings.xml | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 227acf6073..accc637003 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -123,6 +123,54 @@ StaticEmbeddingModel multilingual = StaticEmbeddingModel.loadSentencePiece( +

+ Command Line Tools + + The module ships its own command line launcher, bin/embeddings, next to + bin/opennlp in the binary distribution. Invoked without arguments it lists + the available tools, and every tool prints its help when invoked with the + help parameter. + +
+ Distill Model Tool + + The DistillModel tool compresses a sentence-transformer teacher into a + static embedding table. The teacher is either a Hugging Face model id + (org/model, or org/model@revision to pin a branch, tag, or + commit; the required files download once into a local cache) or a local directory + holding the teacher's tokenizer.json and onnx/model.onnx. + The following command distills a teacher into the directory given by + -out: + + + + -pcaDims is the number of principal components to keep and defaults + to 256. The run ends by assembling the output directory and verifying it with + StaticEmbeddingModel.load, then prints a summary naming the tokenizer + family, the row count, the dimension reduction, and the variance the PCA kept, so a + run that prints a summary is a directory that works. + +
+
+ Assemble Model Tool + + The AssembleModel tool completes a downloaded distillation in place so + StaticEmbeddingModel.load can open it. A Model2Vec distillation writes + model.safetensors, tokenizer.json, and + config.json; for a WordPiece model the tool derives the missing + vocab.txt and tokenizer_config.json from + tokenizer.json, and for a SentencePiece model it checks that the + trained .model file copied from the teacher is present, naming the fix + if it is not. It never overwrites an existing file. + + + + The tool then verifies the directory by loading it and prints the loaded model's + family, row count, and dimension, along with a line for every file it wrote. + +
+
+
The safetensors Reader From d78300e7644fee93a164e098c828ec819e5eda1a Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:32:09 -0400 Subject: [PATCH 63/82] OPENNLP-1877: Fail loud on a config.json declaring non-mean pooling The distiller writes the pooling field but the loader never read it, so a third-party model declaring another pooling silently mean-pooled. Only mean pooling is implemented, so the loader now rejects any other declared value with an InvalidFormatException naming it; a declared mean still loads. --- .../embeddings/StaticEmbeddingModel.java | 16 ++++++++-- .../embeddings/StaticEmbeddingModelTest.java | 30 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index c292241141..b2cfcde9b3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -88,6 +88,8 @@ public enum Normalization { private static final float NORMALIZE_EPSILON = 1e-12f; private static final String WEIGHTS_TENSOR_NAME = "weights"; + // The only pooling this model implements; the value the distiller writes into config.json. + private static final String MEAN_POOLING = "mean"; private static final int[] NO_EXCLUDED_ROWS = new int[0]; // Excluded from neighbor results, including [PAD] and [MASK] that a distilled table keeps. private static final Set WORDPIECE_SPECIAL_TOKENS = @@ -222,14 +224,24 @@ private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, } /** - * Reads the required {@code normalize} switch out of a model's {@code config.json}. + * Reads the required {@code normalize} switch out of a model's {@code config.json}, rejecting + * a configuration whose {@code pooling} field declares anything but the mean pooling this + * model implements. Silently mean-pooling a table distilled for another pooling would produce + * plausible but wrong vectors, so such a model fails loud here. * * @param configFile The {@code config.json} file. * @return The corresponding {@link Normalization}. - * @throws InvalidFormatException Thrown if the field is missing or not a boolean. + * @throws InvalidFormatException Thrown if the {@code normalize} field is missing or not a + * boolean, or the {@code pooling} field declares a pooling other than {@code "mean"}. * @throws IOException Thrown if reading the file fails. */ private static Normalization requiredNormalize(Path configFile) throws IOException { + final String pooling = FlatJsonFields.topLevelString(configFile, "pooling"); + if (pooling != null && !MEAN_POOLING.equals(pooling)) { + throw new InvalidFormatException(configFile + " declares pooling '" + pooling + + "' but only '" + MEAN_POOLING + "' pooling is implemented; embedding this model " + + "would silently pool differently than its distiller intended"); + } final Boolean normalize = FlatJsonFields.topLevelBoolean(configFile, "normalize"); if (normalize == null) { throw new InvalidFormatException(configFile + " has no boolean 'normalize' field; " diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index 9ebfbfc187..377de083e7 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -365,6 +365,36 @@ void testDirectoryLoadRejectsAConfigWithoutNormalize(@TempDir Path dir) throws I assertTrue(e.getMessage().contains("normalize")); } + @Test + void testDirectoryLoadRejectsAConfigDeclaringNonMeanPooling(@TempDir Path dir) + throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + writeConfigs(dir, "false", "true"); + // Only mean pooling is implemented; a third-party config declaring another pooling must + // fail loud instead of silently mean-pooling with the wrong semantics. + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false,\"pooling\":\"max\"}"); + + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); + assertTrue(e.getMessage().contains("max"), e.getMessage()); + assertTrue(e.getMessage().contains("mean"), e.getMessage()); + } + + @Test + void testDirectoryLoadAcceptsTheDeclaredMeanPooling(@TempDir Path dir) throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + writeConfigs(dir, "false", "true"); + // The pooling the distiller writes; declaring it explicitly must load like omitting it. + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false,\"pooling\":\"mean\"}"); + + assertArrayEquals(new float[] {3.5f, 35f, 350f}, + StaticEmbeddingModel.load(dir).embed("hello world"), 1e-5f); + } + @Test void testDirectoryLoadRejectsContradictoryStripAccents(@TempDir Path dir) throws IOException { writeVocab(dir); From b42043ce6fe7f56a181304898ecabbc98571691b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:32:55 -0400 Subject: [PATCH 64/82] OPENNLP-1877: Clamp the neighbor scan's topK to the vocabulary size mostSimilar and analogy sized the candidate arrays by the raw topK before any clamping, so mostSimilar(word, Integer.MAX_VALUE) failed with an OutOfMemoryError. The scan can never yield more than one neighbor per row, so the capacity is now the smaller of topK and the row count. --- .../embeddings/StaticEmbeddingModel.java | 6 ++++-- .../StaticEmbeddingModelSimilarityTest.java | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index b2cfcde9b3..45f48d98ee 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -762,9 +762,11 @@ private List nearestNeighbors(float[] query, int topK, int[] sortedExc if (queryNorm < NORMALIZE_EPSILON) { return List.of(); } - final TopK best = new TopK(topK); - int nextExcluded = 0; final int rowCount = rowNorms.length; + // The capacity sizes the candidate arrays; a topK beyond the vocabulary (the scan can never + // yield more than every row) would otherwise allocate topK-sized arrays or overflow. + final TopK best = new TopK(Math.min(topK, rowCount)); + int nextExcluded = 0; for (int row = 0; row < rowCount; row++) { if (nextExcluded < sortedExcludedRows.length && sortedExcludedRows[nextExcluded] == row) { nextExcluded++; diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java index d07ff9481f..00d872e238 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSimilarityTest.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.List; import org.junit.jupiter.api.Test; @@ -30,6 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -106,6 +108,22 @@ void testMostSimilarExcludesSpecialTokensAndOrdersByDescendingSimilarity(@TempDi assertEquals("apple", result.get(result.size() - 1).token()); } + @Test + void testMostSimilarClampsTopKToTheVocabularySize(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = load(dir); + + // topK sizes the candidate arrays, so it must be clamped to the vocabulary before + // allocation; unclamped, Integer.MAX_VALUE fails with OutOfMemoryError. The fixture has + // 8 rows, 3 of them special, so any request larger than the vocabulary returns the same + // 5 neighbors a topK of 8 would. + final List result = assertTimeoutPreemptively(Duration.ofSeconds(10), + () -> model.mostSimilar("king", Integer.MAX_VALUE)); + + assertEquals(model.vocabularySize() - 3, result.size()); + assertEquals("king", result.get(0).token()); + assertEquals(result, model.mostSimilar("king", model.vocabularySize())); + } + @Test void testMostSimilarOfZeroVectorQueryReturnsEmptyList(@TempDir Path dir) throws IOException { final StaticEmbeddingModel model = load(dir); From 86963bc82a9b5207270fb67ec9ca6d1fc5ed93ab Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:34:00 -0400 Subject: [PATCH 65/82] OPENNLP-1877: Reject non-finite embedding matrix values at load time The distiller path zeroes non-finite teacher values, but the loader accepted any bytes, and a single NaN row defeats the zero-norm guard and the TopK comparison, silently corrupting similarity rankings. Loading a matrix that holds a NaN or infinity now throws InvalidFormatException naming the row. --- .../embeddings/StaticEmbeddingModel.java | 12 +++++++- .../embeddings/StaticEmbeddingModelTest.java | 30 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 45f48d98ee..12f5377e3e 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -480,7 +480,7 @@ private record Matrix(float[] embeddings, float[] weights, int dimension) { * @param vocabularySourceName The vocabulary's source, for error messages. * @return The matrix, its optional weights, and its dimension. * @throws InvalidFormatException Thrown if the matrix's row count or the weights tensor's - * length disagrees with the vocabulary size. + * length disagrees with the vocabulary size, or the matrix contains a non-finite value. * @throws IOException Thrown if reading the file fails. */ private static Matrix readMatrix(EmbeddingVocabulary vocabulary, Path safetensorsFile, @@ -496,6 +496,16 @@ private static Matrix readMatrix(EmbeddingVocabulary vocabulary, Path safetensor } final int dimension = matrixInfo.shape()[1]; final float[] embeddings = tensors.readFloats(matrixName); + // Distillation replaces non-finite teacher values with zero before writing, so a NaN or + // infinity here marks a corrupt or foreign file; unrejected, a single NaN row silently + // defeats the norm guards and corrupts every similarity ranking it appears in. + for (int i = 0; i < embeddings.length; i++) { + if (!Float.isFinite(embeddings[i])) { + throw new InvalidFormatException("Embedding matrix '" + matrixName + "' in " + + safetensorsFile + " holds the non-finite value " + embeddings[i] + " in row " + + (i / dimension) + "; the matrix is corrupt"); + } + } float[] weights = null; if (tensors.tensorNames().contains(WEIGHTS_TENSOR_NAME)) { diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index 377de083e7..e122918dfe 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -279,6 +279,36 @@ void testLoadRejectsVocabularySizeMismatch(@TempDir Path dir) throws IOException assertTrue(e.getMessage().contains("rows")); } + @Test + void testLoadRejectsANonFiniteMatrixValue(@TempDir Path dir) throws IOException { + // The distiller replaces non-finite teacher values with zero before writing, so a NaN in a + // loaded matrix marks a corrupt or foreign file. It must fail loud at load time: a NaN row + // defeats both the zero-norm guard and every similarity comparison downstream. + final float[][] rows = new float[ROWS.length][]; + for (int r = 0; r < ROWS.length; r++) { + rows[r] = ROWS[r].clone(); + } + rows[4][1] = Float.NaN; + final Path vocab = writeVocab(dir); + final Path nanTensors = dir.resolve("nan.safetensors"); + SafetensorsTestFiles.write(nanTensors, SafetensorsTestFiles.matrix("embeddings", rows)); + + final InvalidFormatException nan = assertThrows(InvalidFormatException.class, + () -> StaticEmbeddingModel.load(vocab, nanTensors, Casing.UNCASED, Normalization.NONE)); + assertTrue(nan.getMessage().contains("row 4"), nan.getMessage()); + + // An infinity is just as corrupting and must be rejected the same way. + rows[4][1] = Float.POSITIVE_INFINITY; + final Path infiniteTensors = dir.resolve("infinite.safetensors"); + SafetensorsTestFiles.write(infiniteTensors, + SafetensorsTestFiles.matrix("embeddings", rows)); + + final InvalidFormatException infinite = assertThrows(InvalidFormatException.class, + () -> StaticEmbeddingModel.load(vocab, infiniteTensors, + Casing.UNCASED, Normalization.NONE)); + assertTrue(infinite.getMessage().contains("row 4"), infinite.getMessage()); + } + @Test void testLoadRejectsWeightsSizeMismatch(@TempDir Path dir) throws IOException { // A weights tensor sized for a different (smaller) vocabulary than the embedding matrix. From 52ac8141162725897cabf9c16010531001605c8e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:36:36 -0400 Subject: [PATCH 66/82] OPENNLP-1877: Exercise Casing.CASED through the directory loader do_lower_case=false was never tested; the new case asserts that a cased model matches lower-case vocabulary entries as-is and folds upper-case text to the skipped unknown token instead of lower-casing it first. --- .../embeddings/StaticEmbeddingModelTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java index e122918dfe..c721f3985a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTest.java @@ -349,6 +349,21 @@ void testLoadsFromAModelDirectory(@TempDir Path dir) throws IOException { assertArrayEquals(new float[] {3.5f, 35f, 350f}, model.embed("HELLO WORLD"), 1e-5f); } + @Test + void testDirectoryLoadReadsCasedFromTheTokenizerConfig(@TempDir Path dir) throws IOException { + writeVocab(dir); + writeSafetensors(dir, false); + writeConfigs(dir, "false", "false"); + + final StaticEmbeddingModel model = StaticEmbeddingModel.load(dir); + + // do_lower_case=false maps to Casing.CASED: lower-case text still matches the vocabulary... + assertArrayEquals(new float[] {3.5f, 35f, 350f}, model.embed("hello world"), 1e-5f); + // ...but upper-case text is preserved as-is, matches no cased vocabulary entry, folds to + // the (skipped) [UNK], and pools to the zero vector instead of being lower-cased first. + assertArrayEquals(new float[] {0f, 0f, 0f}, model.embed("HELLO WORLD"), 1e-5f); + } + @Test void testDirectoryLoadReadsNormalizeFromTheConfig(@TempDir Path dir) throws IOException { writeVocab(dir); From b0ae5e7f71798acb91e444ecc7367a3a40d238e7 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:37:14 -0400 Subject: [PATCH 67/82] OPENNLP-1877: Mirror the manual's explicit-overload listing with a test The chapter's second programlisting (the explicit load and loadSentencePiece overloads) had no test pinning it. testExplicitOverloads loads both fixture layouts exactly as the listing shows, backed by a new minimal SentencePiece directory fixture, and the chapter now cites the test the way it already cites the directory-load listing. --- opennlp-docs/src/docbkx/embeddings.xml | 4 ++ .../embeddings/EmbeddingTestFixtures.java | 57 +++++++++++++++++++ .../StaticEmbeddingUsageExampleTest.java | 47 ++++++++++++++- 3 files changed, 105 insertions(+), 3 deletions(-) diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index accc637003..5c53a63c74 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -105,6 +105,10 @@ StaticEmbeddingModel multilingual = StaticEmbeddingModel.loadSentencePiece( StaticEmbeddingModel.Normalization.L2); ]]> + + The testExplicitOverloads case of + StaticEmbeddingUsageExampleTest asserts the two overloads shown here. + Matrix rows are resolved by piece string, never by tokenizer id, because the two files of a SentencePiece model routinely order and offset their ids differently. A diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java index 53e1ffce46..8a3804b320 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java @@ -16,13 +16,17 @@ */ package opennlp.embeddings; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import opennlp.embeddings.StaticEmbeddingModel.Casing; import opennlp.embeddings.StaticEmbeddingModel.Normalization; +import opennlp.subword.sentencepiece.SentencePieceTokenizer; /** * Fixtures shared by more than one test in this module: the small WordPiece table the geometry @@ -97,6 +101,59 @@ private static void writeVocabularyAndMatrix(Path dir) throws IOException { SafetensorsTestFiles.matrix("embeddings", ANALOGY_ROWS)); } + /** The classpath resource of the tiny trained SentencePiece model shared by the tests. */ + static final String TINY_UNIGRAM_RESOURCE = "/opennlp/embeddings/tiny-unigram.model"; + + /** The row width of the matrix {@link #writeSentencePieceDirectory(Path)} writes. */ + static final int SENTENCEPIECE_DIMENSION = 4; + + /** + * Writes a minimal loadable SentencePiece model into a directory: the trained + * {@code tiny-unigram.model} fixture copied as {@code sentencepiece.bpe.model}, a Unigram + * {@code tokenizer.json} whose vocabulary is the unknown piece followed by every poolable + * tokenizer piece, and a deterministic embedding matrix with one row per listed piece. A test + * can then load it through the explicit + * {@code StaticEmbeddingModel.loadSentencePiece(Path, Path, Path, Normalization)} overload + * the way the manual's listing shows. + * + * @param dir The directory to write the model files into. + * @throws IOException Thrown if reading the fixture resource or writing a file fails. + */ + static void writeSentencePieceDirectory(Path dir) throws IOException { + final byte[] modelBytes; + try (InputStream in = + EmbeddingTestFixtures.class.getResourceAsStream(TINY_UNIGRAM_RESOURCE)) { + modelBytes = in.readAllBytes(); + } + Files.write(dir.resolve("sentencepiece.bpe.model"), modelBytes); + final SentencePieceTokenizer tokenizer = + SentencePieceTokenizer.load(new ByteArrayInputStream(modelBytes)); + final List rows = new ArrayList<>(); + rows.add(""); + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + if (!tokenizer.isControl(id) && !tokenizer.isUnknown(id)) { + rows.add(tokenizer.idToPiece(id)); + } + } + final StringBuilder json = + new StringBuilder("{\"model\":{\"type\":\"Unigram\",\"unk_id\":0,\"vocab\":["); + for (int i = 0; i < rows.size(); i++) { + if (i > 0) { + json.append(','); + } + json.append('[').append(jsonString(rows.get(i))).append(",-1.5]"); + } + Files.writeString(dir.resolve("tokenizer.json"), json.append("]}}").toString()); + final float[][] matrix = new float[rows.size()][SENTENCEPIECE_DIMENSION]; + for (int row = 0; row < matrix.length; row++) { + for (int d = 0; d < SENTENCEPIECE_DIMENSION; d++) { + matrix[row][d] = row + d * 0.25f; + } + } + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", matrix)); + } + /** * {@return {@code value} as a JSON string literal, quoted and escaped} * diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java index ae8278ce31..ecd8513358 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingUsageExampleTest.java @@ -27,9 +27,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Pins the cookbook path documented in {@code embeddings.xml}, mirroring its usage listing: - * load a model directory with {@link StaticEmbeddingModel#load(Path)}, embed a text, and call - * {@code similarity}, {@code mostSimilar}, and {@code analogy}. + * Pins the cookbook paths documented in {@code embeddings.xml}, mirroring its listings: load a + * model directory with {@link StaticEmbeddingModel#load(Path)}, embed a text, and call + * {@code similarity}, {@code mostSimilar}, and {@code analogy}; and load through the explicit + * WordPiece and SentencePiece overloads. */ public class StaticEmbeddingUsageExampleTest { @@ -52,4 +53,44 @@ void testEmbedSimilarityNeighborsAndAnalogy(@TempDir Path dir) throws IOExceptio assertEquals(1, analogy.size()); assertEquals("queen", analogy.get(0).token()); } + + @Test + void testExplicitOverloads(@TempDir Path wordPieceDir, @TempDir Path sentencePieceDir) + throws IOException { + EmbeddingTestFixtures.writeAnalogyDirectory(wordPieceDir); + EmbeddingTestFixtures.writeSentencePieceDirectory(sentencePieceDir); + + // The manual's explicit WordPiece overload: the data files plus the two switches the + // model's configuration publishes. + final StaticEmbeddingModel model = StaticEmbeddingModel.load( + wordPieceDir.resolve("vocab.txt"), wordPieceDir.resolve("model.safetensors"), + StaticEmbeddingModel.Casing.UNCASED, + StaticEmbeddingModel.Normalization.L2); + assertEquals(2, model.dimension()); + assertUnitLength(model.embed("king")); + + // The manual's explicit SentencePiece overload: no casing switch, because the trained + // .model file carries the model's own text normalizer. + final StaticEmbeddingModel multilingual = StaticEmbeddingModel.loadSentencePiece( + sentencePieceDir.resolve("sentencepiece.bpe.model"), + sentencePieceDir.resolve("tokenizer.json"), + sentencePieceDir.resolve("model.safetensors"), + StaticEmbeddingModel.Normalization.L2); + assertEquals(EmbeddingTestFixtures.SENTENCEPIECE_DIMENSION, multilingual.dimension()); + assertUnitLength(multilingual.embed("a")); + } + + /** + * Asserts that a vector has unit L2 length, the visible effect of choosing + * {@code Normalization.L2} in the explicit overloads. + * + * @param vector The vector to measure. + */ + private static void assertUnitLength(float[] vector) { + double normSquared = 0; + for (final float v : vector) { + normSquared += (double) v * v; + } + assertEquals(1.0, Math.sqrt(normSquared), 1e-5); + } } From 851196f4dabded36b7d1a21b48ee1f67bf09b35d Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:37:29 -0400 Subject: [PATCH 68/82] OPENNLP-1877: Document semantic search in the manual with a mirrored test The headline use case, ranking documents against a query by cosine similarity, existed only in the module README. The chapter gains a Semantic Search section whose listing scores each document with the public similarity method and sorts by descending score, and the new StaticEmbeddingSearchExampleTest asserts the exact ranking on the analogy fixture's known geometry. --- opennlp-docs/src/docbkx/embeddings.xml | 33 ++++++++++ .../StaticEmbeddingSearchExampleTest.java | 63 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingSearchExampleTest.java diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 5c53a63c74..825e8b22d9 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -127,6 +127,39 @@ StaticEmbeddingModel multilingual = StaticEmbeddingModel.loadSentencePiece(
+
+ Semantic Search + + The headline use case is ranking documents against a query by meaning rather than by + shared words. similarity embeds both texts and returns the cosine + similarity of their vectors, so a small document list ranks with one call per + document: + + + documents = List.of( + "How do I brew espresso at home?", + "The history of tea in East Asia", + "Best grinders for pour-over coffee"); + +record Scored(String document, double score) {} +List results = new ArrayList<>(); +for (String document : documents) { + results.add(new Scored(document, model.similarity(query, document))); +} +results.sort(Comparator.comparingDouble(Scored::score).reversed());]]> + + + StaticEmbeddingSearchExampleTest asserts the ranking shown here. For a + corpus too large to score per query, embed each document once with + embed, keep the vectors in any vector index, and embed only the query + at search time; the index does not care how the vectors were produced. + +
+
Command Line Tools diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingSearchExampleTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingSearchExampleTest.java new file mode 100644 index 0000000000..3538ed6608 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingSearchExampleTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins the semantic-search listing documented in {@code embeddings.xml}: score every document + * against a query with {@link StaticEmbeddingModel#similarity(String, String)} and sort by + * descending score, asserting the exact ranking on the analogy fixture's known geometry. + */ +public class StaticEmbeddingSearchExampleTest { + + /** A scored document, as the manual's listing declares it. */ + record Scored(String document, double score) { + } + + @Test + void testRanksDocumentsByCosineSimilarityToTheQuery(@TempDir Path dir) throws IOException { + EmbeddingTestFixtures.writeAnalogyDirectory(dir); + final StaticEmbeddingModel model = StaticEmbeddingModel.load(dir); + + final String query = "king"; + final List documents = List.of("queen woman", "apple", "king man"); + + final List results = new ArrayList<>(); + for (final String document : documents) { + results.add(new Scored(document, model.similarity(query, document))); + } + results.sort(Comparator.comparingDouble(Scored::score).reversed()); + + // The fixture's mean-pooled vectors give distinct cosines to "king" ([3,3]): "king man" + // pools to [2.5,2] (0.994), "queen woman" to [1.5,3] (0.949), "apple" is [-3,-1] (-0.894). + assertEquals(List.of("king man", "queen woman", "apple"), + results.stream().map(Scored::document).toList()); + assertTrue(results.get(0).score() > results.get(1).score()); + assertTrue(results.get(2).score() < 0); + } +} From 0e1b0fa156d56412fed584db0c9bf344a3738577 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 16 Aug 2026 06:15:21 -0400 Subject: [PATCH 69/82] OPENNLP-1877: Parse teacher references and hex digests with cursor scans The hub cache compiled two Patterns for the org/model@revision reference and the hex shape of commit shas and digests. Hand scans over the same ASCII grammars replace them; the module's parsing is now regex-free throughout. --- .../embeddings/HuggingFaceModelCache.java | 92 ++++++++++++++++--- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java index 1e61977fc7..65cfd6cfc3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HuggingFaceModelCache.java @@ -35,8 +35,6 @@ import java.util.HexFormat; import java.util.List; import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Fetches the files a distillation needs from a Hugging Face model repository into a local cache @@ -74,10 +72,14 @@ final class HuggingFaceModelCache { private static final String DEFAULT_REVISION = "main"; /** - * A teacher reference: an organization and a model name, both of word characters, dots, or - * dashes, optionally followed by {@code @} and the revision to pin. + * A parsed teacher reference: an organization and a model name joined by {@code /}, optionally + * followed by {@code @} and the revision to pin; see {@link #parseTeacherReference(String)}. + * + * @param modelId The {@code org/model} id. + * @param revision The pinned revision, or {@code null} when none is given. */ - private static final Pattern TEACHER_PATTERN = Pattern.compile("([\\w.-]+/[\\w.-]+)(?:@([\\w.-]+))?"); + private record TeacherReference(String modelId, String revision) { + } /** The directory the cache lives in, below the user's home directory. */ private static final String CACHE_DIRECTORY = ".cache"; @@ -110,8 +112,6 @@ final class HuggingFaceModelCache { /** The length in hex of a SHA-256: the shape of the digest published for a Git LFS file. */ private static final int SHA256_HEX_LENGTH = 64; - /** A hex string of any length, the shape both the commit sha and the digests have. */ - private static final Pattern HEX_PATTERN = Pattern.compile("[0-9a-fA-F]+"); /** The header git hashes in front of a blob's bytes, completed by the length and a NUL byte. */ private static final String GIT_BLOB_PREFIX = "blob "; @@ -208,13 +208,13 @@ static Path resolve(String teacher, String hubBase, Path cacheRoot, if (Files.isDirectory(local)) { return local; } - final Matcher reference = TEACHER_PATTERN.matcher(teacher); - if (!reference.matches()) { + final TeacherReference reference = parseTeacherReference(teacher); + if (reference == null) { throw new IllegalArgumentException("Teacher '" + teacher + "' is neither a local " + "directory nor a Hugging Face model id (expected 'org/model' or 'org/model@revision')"); } - final String modelId = reference.group(1); - final String requestedRevision = reference.group(2); + final String modelId = reference.modelId(); + final String requestedRevision = reference.revision(); final Path cache = cacheRoot.resolve(cacheDirectoryName(teacher)); final String pinned = pinnedRevision(cache); if (pinned != null && hasRequiredFiles(cache) @@ -494,8 +494,72 @@ private static String originHeader(HttpResponse response, String na * @param value The value to check; may be {@code null}. */ private static boolean isCommitSha(String value) { - return value != null && value.length() == SHA1_HEX_LENGTH - && HEX_PATTERN.matcher(value).matches(); + return value != null && value.length() == SHA1_HEX_LENGTH && isHex(value); + } + + /** + * {@return whether a value is one or more ASCII hex characters, the shape both the commit sha + * and the digests have} + * + * @param value The value to check. + */ + private static boolean isHex(String value) { + if (value.isEmpty()) { + return false; + } + for (int i = 0; i < value.length(); i++) { + final char c = value.charAt(i); + final boolean hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F'); + if (!hex) { + return false; + } + } + return true; + } + + /** + * Parses a teacher reference: an organization and a model name, both runs of ASCII word + * characters, dots, or dashes, joined by {@code /} and optionally followed by {@code @} and a + * revision of the same shape. + * + * @param teacher The reference to parse. + * @return The parsed reference, or {@code null} when the value does not have this form. + */ + private static TeacherReference parseTeacherReference(String teacher) { + final int slash = teacher.indexOf('/'); + if (slash < 0) { + return null; + } + final int at = teacher.indexOf('@', slash + 1); + final String organization = teacher.substring(0, slash); + final String model = at < 0 ? teacher.substring(slash + 1) : teacher.substring(slash + 1, at); + final String revision = at < 0 ? null : teacher.substring(at + 1); + if (!isReferencePart(organization) || !isReferencePart(model) + || (revision != null && !isReferencePart(revision))) { + return null; + } + return new TeacherReference(organization + "/" + model, revision); + } + + /** + * {@return whether a reference part is one or more ASCII word characters, dots, or dashes} + * + * @param part The part to check. + */ + private static boolean isReferencePart(String part) { + if (part.isEmpty()) { + return false; + } + for (int i = 0; i < part.length(); i++) { + final char c = part.charAt(i); + final boolean allowed = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') || c == '_' || c == '.' || c == '-'; + if (!allowed) { + return false; + } + } + return true; } /** @@ -582,7 +646,7 @@ private enum Checksum { */ static Checksum of(String value) { for (final Checksum checksum : values()) { - if (value.length() == checksum.hexLength && HEX_PATTERN.matcher(value).matches()) { + if (value.length() == checksum.hexLength && isHex(value)) { return checksum; } } From 5cc79e3490ffb55dae1fcdc3181a9af768b9dfb2 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 16 Aug 2026 06:15:31 -0400 Subject: [PATCH 70/82] OPENNLP-1877: Distill whole terms as extra rows matched before subword pieces A distillation can now take a term list, a learned corpus vocabulary of whole words and multi-word phrases. Each term is segmented by the teacher's own tokenizer, encoded through the teacher as one sequence, and appended to the table after the subword rows, recorded as terms.txt in the model directory. The same PCA and Zipf pipeline spans all rows, and a term equal to a surviving vocabulary token is dropped as a duplicate row. At embed time the model matches text against its terms greedily longest-first over case-folded word runs (StringUtil.toLowerCase, one code point to one code point) and pools a matched term's single row in place of its words' subword pieces; text between matches tokenizes as before, and a model without a terms file embeds exactly as it did. Terms are neighbor candidates in mostSimilar like any token. The DistillModel tool gains -terms, documented in the manual. TeacherTokenizer's two Patterns (the unused-token filter and the template splitter) are replaced with cursor scans along the way. --- opennlp-docs/src/docbkx/embeddings.xml | 18 +- .../opennlp/embeddings/ModelAssembler.java | 61 +---- .../opennlp/embeddings/ModelDistiller.java | 207 +++++++++++++- .../opennlp/embeddings/ModelFileNames.java | 3 + .../embeddings/StaticEmbeddingModel.java | 240 +++++++++++++---- .../opennlp/embeddings/TeacherTokenizer.java | 164 ++++++++++- .../opennlp/embeddings/TermSegmenter.java | 129 +++++++++ .../java/opennlp/embeddings/TermTable.java | 254 ++++++++++++++++++ .../embeddings/cmdline/AssembleModelTool.java | 4 +- .../cmdline/DistillModelParams.java | 10 + .../embeddings/cmdline/DistillModelTool.java | 31 ++- .../embeddings/EmbeddingTestFixtures.java | 19 +- .../embeddings/ModelDistillerTest.java | 27 ++ .../StaticEmbeddingModelTermTest.java | 187 +++++++++++++ .../embeddings/TeacherTokenizerTest.java | 22 ++ .../opennlp/embeddings/TermSegmenterTest.java | 131 +++++++++ .../opennlp/embeddings/TermTableTest.java | 155 +++++++++++ .../opennlp/embeddings/cmdline/CLITest.java | 3 +- 18 files changed, 1523 insertions(+), 142 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermSegmenter.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermTable.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTermTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermSegmenterTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermTableTest.java diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 825e8b22d9..69fd85d1b9 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -179,7 +179,7 @@ results.sort(Comparator.comparingDouble(Scored::score).reversed());]]> The following command distills a teacher into the directory given by -out: - + -pcaDims is the number of principal components to keep and defaults to 256. The run ends by assembling the output directory and verifying it with @@ -187,6 +187,22 @@ results.sort(Comparator.comparingDouble(Scored::score).reversed());]]> family, the row count, the dimension reduction, and the variance the PCA kept, so a run that prints a summary is a directory that works. + + -terms names an optional term file: one term per line, with text after + a tab ignored, so a learned vocabulary TSV works unchanged. Each term, a whole word + or a multi-word phrase such as a domain vocabulary entry, is segmented by the + teacher's own tokenizer, encoded through the teacher as one sequence, and appended + to the table as an extra row, recorded in the model directory as + terms.txt. When such a model embeds text, it first matches the text + against its terms greedily longest-first, case-insensitively, and pools a matched + term's single row instead of the subword pieces of its words; text between matches + is tokenized as usual, and a model without a term file embeds exactly as before. + Terms should arrive sorted by descending corpus frequency, because the Zipf + weighting treats the subword rows and the term rows as one frequency ranking. A + term that equals a vocabulary token is dropped as a duplicate row, and terms are + also returned by the similarity search of mostSimilar like any + vocabulary token. +
Assemble Model Tool diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java index 74cdfbf72c..a4e64672ee 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java @@ -70,11 +70,13 @@ private ModelAssembler() { * * @param family {@code "WordPiece"} or {@code "SentencePiece"}. * @param dimension The embedding dimension of the loaded model. - * @param vocabularySize The number of rows in the loaded model's table. + * @param vocabularySize The number of subword rows in the loaded model's table. + * @param termCount The number of term rows after the subword rows; {@code 0} for a + * model without a term table. * @param wroteVocabulary Whether a {@code vocab.txt} was written. * @param wroteTokenizerConfig Whether a {@code tokenizer_config.json} was written. */ - public record Result(String family, int dimension, int vocabularySize, + public record Result(String family, int dimension, int vocabularySize, int termCount, boolean wroteVocabulary, boolean wroteTokenizerConfig) { } @@ -147,7 +149,7 @@ private static Result assembleWordpiece(Path modelDirectory, TokenizerJson token } final StaticEmbeddingModel model = load(modelDirectory); return new Result(FAMILY_WORDPIECE, model.dimension(), model.vocabularySize(), - wroteVocabulary, wroteTokenizerConfig); + model.termCount(), wroteVocabulary, wroteTokenizerConfig); } /** @@ -169,7 +171,7 @@ private static Result assembleSentencePiece(Path modelDirectory) throws IOExcept } final StaticEmbeddingModel model = load(modelDirectory); return new Result(FAMILY_SENTENCEPIECE, model.dimension(), model.vocabularySize(), - false, false); + model.termCount(), false, false); } /** @@ -253,7 +255,7 @@ private static TokenizerJson readTokenizerJson(Path file) throws IOException { modelType = model.type(); orderedVocabulary = model.orderedVocabulary(); } - case "normalizer" -> lowerCase = parseNormalizerLowercase(cursor); + case "normalizer" -> lowerCase = TeacherTokenizer.parseNormalizerLowercase(cursor); default -> cursor.skipValue(); } cursor.skipWhitespace(); @@ -370,53 +372,4 @@ private static List parseVocabularyDictionary(JsonCursor cursor) return ordered; } - /** - * Reads the flat {@code lowercase} boolean of a {@code normalizer} object, for the BERT - * normalizer a WordPiece distillation carries. - * - * @param cursor The cursor, positioned at the normalizer value. - * @return The {@code lowercase} flag, or {@code null} when the value is JSON null or the flag is - * absent (for example a nested normalizer with no flat flag). - */ - private static Boolean parseNormalizerLowercase(JsonCursor cursor) - throws InvalidFormatException { - if (cursor.peek() != '{') { - cursor.skipValue(); - return null; - } - cursor.expect('{'); - cursor.skipWhitespace(); - Boolean lowerCase = null; - if (cursor.peek() == '}') { - cursor.consume(); - return null; - } - while (true) { - cursor.skipWhitespace(); - final String key = cursor.parseString(); - cursor.skipWhitespace(); - cursor.expect(':'); - cursor.skipWhitespace(); - if ("lowercase".equals(key)) { - if (cursor.consumeLiteral("true")) { - lowerCase = Boolean.TRUE; - } else if (cursor.consumeLiteral("false")) { - lowerCase = Boolean.FALSE; - } else { - cursor.skipValue(); - } - } else { - cursor.skipValue(); - } - cursor.skipWhitespace(); - final char next = cursor.consume(); - if (next == ',') { - continue; - } - if (next == '}') { - return lowerCase; - } - throw cursor.malformed("Expected ',' or '}' after a normalizer field, got '" + next + "'"); - } - } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java index 18a8a87aa3..8aa207a8e5 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelDistiller.java @@ -20,6 +20,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; import opennlp.tools.util.java.Experimental; @@ -84,13 +91,14 @@ public interface ProgressListener { * the variance the PCA kept. * * @param family {@code "WordPiece"} or {@code "SentencePiece"}. - * @param vocabularySize The number of rows in the distilled table. + * @param vocabularySize The number of subword rows in the distilled table. + * @param termCount The number of term rows appended after the subword rows. * @param teacherDimension The teacher's hidden dimension. * @param dimension The distilled table's dimension (after PCA). * @param explainedVarianceRatio The share of the embedding variance the PCA kept. */ - public record Result(String family, int vocabularySize, int teacherDimension, int dimension, - double explainedVarianceRatio) { + public record Result(String family, int vocabularySize, int termCount, int teacherDimension, + int dimension, double explainedVarianceRatio) { } /** @@ -111,9 +119,34 @@ public record Result(String family, int vocabularySize, int teacherDimension, in */ public static Result distill(String teacher, Path outputDirectory, int pcaDims, ProgressListener listener) throws IOException { + return distill(teacher, outputDirectory, pcaDims, List.of(), listener); + } + + /** + * Distills a teacher into a model directory with additional term rows, resolving the teacher + * reference the way {@link #distill(String, Path, int, ProgressListener)} does. + * + * @param teacher The teacher: a local directory or a Hugging Face model id. Must not + * be {@code null}. + * @param outputDirectory The model directory to write. Must not be {@code null}. + * @param pcaDims The number of principal components to keep. + * @param terms The terms to distill as extra rows; see + * {@link #distill(Path, Path, int, List, ProgressListener)}. Must not + * be {@code null}. + * @param listener Receives progress lines; may be {@code null}. + * @return The distillation result, read back from the verified directory. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or invalid, a term + * normalizes to nothing, the teacher reference is malformed, or the teacher cannot be run. + * @throws IOException Thrown if reading or writing a file fails, or if a teacher cannot be + * downloaded and verified. + */ + public static Result distill(String teacher, Path outputDirectory, int pcaDims, + List terms, ProgressListener listener) + throws IOException { checkOutput(outputDirectory, pcaDims); + final List prepared = prepareTerms(terms); return distill(HuggingFaceModelCache.resolve(teacher, listener), outputDirectory, pcaDims, - listener); + prepared, listener); } /** @@ -142,6 +175,43 @@ public static Result distill(String teacher, Path outputDirectory, int pcaDims, public static Result distill(Path teacherDirectory, Path outputDirectory, int pcaDims, ProgressListener listener) throws IOException { + return distill(teacherDirectory, outputDirectory, pcaDims, List.of(), listener); + } + + /** + * Distills a teacher into a model directory with additional term rows: whole words and + * multi-word phrases (a learned corpus vocabulary) that are segmented by the teacher's own + * tokenizer, run through the teacher as full sequences, and appended to the table after the + * subword rows. The loaded model then matches text against these terms greedily + * longest-first before falling back to subword pieces. + * + *

Each term is normalized to lower-cased words joined by single spaces before use; terms + * that normalize to the same form are distilled once, and a term equal to a surviving + * vocabulary token is dropped, because its row would duplicate that token's. The terms are + * written to the model directory as {@code terms.txt}, one per line in row order, and should + * arrive sorted by descending corpus frequency: the Zipf weighting spans the subword rows and + * the term rows as one ranking.

+ * + * @param teacherDirectory The teacher's directory, as in + * {@link #distill(Path, Path, int, ProgressListener)}. A Unigram + * teacher must also hold its trained SentencePiece {@code .model} + * file. Must not be {@code null}. + * @param outputDirectory The model directory to write, as in + * {@link #distill(Path, Path, int, ProgressListener)}. Must not be + * {@code null}. + * @param pcaDims The number of principal components to keep. + * @param terms The terms to distill as extra rows; empty for none. Must not be + * {@code null} and must not contain {@code null}. + * @param listener Receives progress lines; may be {@code null}. + * @return The distillation result, read back from the verified directory. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or invalid, a term + * normalizes to nothing, the teacher directory lacks its files, or the teacher cannot be + * run. + * @throws IOException Thrown if reading or writing a file fails. + */ + public static Result distill(Path teacherDirectory, Path outputDirectory, int pcaDims, + List terms, ProgressListener listener) + throws IOException { if (teacherDirectory == null) { throw new IllegalArgumentException("TeacherDirectory must not be null"); } @@ -164,6 +234,23 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc throw new IllegalArgumentException("Teacher directory " + teacherDirectory + " has no " + "vocabulary token left after cleaning; there is nothing to distill"); } + final List termList = new ArrayList<>(prepareTerms(terms)); + if (!termList.isEmpty()) { + // A term equal to a surviving vocabulary token would encode to the same teacher sequence + // and duplicate that token's row, so it is dropped; matching then reaches the token's row + // through the subword fallback instead. + final Set keptTokens = new HashSet<>(rows * 2); + for (int row = 0; row < rows; row++) { + keptTokens.add(tokenizer.rowToken(row)); + } + final int requestedTerms = termList.size(); + termList.removeIf(keptTokens::contains); + if (requestedTerms > termList.size()) { + report(listener, "Dropped " + (requestedTerms - termList.size()) + + " terms already present as vocabulary tokens"); + } + } + final int totalRows = rows + termList.size(); report(listener, "Encoding " + rows + " vocabulary tokens of " + teacherDirectory + " through its ONNX graph"); @@ -172,7 +259,7 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc try (OnnxTeacherEncoder encoder = OnnxTeacherEncoder.load(onnxFile)) { float[][] first = encoder.encodeBatch(new long[][] {tokenizer.inputSequence(0)}); teacherDimension = first[0].length; - embeddings = new float[rows * teacherDimension]; + embeddings = new float[totalRows * teacherDimension]; System.arraycopy(first[0], 0, embeddings, 0, teacherDimension); int row = 1; while (row < rows) { @@ -189,6 +276,8 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc row += batchSize; report(listener, "Encoded " + row + " / " + rows + " vocabulary tokens"); } + encodeTerms(termList, tokenizer, teacherDirectory, encoder, embeddings, rows, + teacherDimension, listener); } nonFiniteToZero(embeddings); @@ -196,22 +285,22 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc final float[] transformed; final int components; double explainedVarianceRatio = 1.0; - if (requested >= rows) { + if (requested >= totalRows) { // A PCA with more components than rows is not a reduction; Model2Vec skips it with a // warning. Only reachable for toy vocabularies, which then keep the teacher's dimension. transformed = embeddings; components = teacherDimension; } else { - report(listener, "Reducing " + rows + " x " + teacherDimension + " to " + requested + report(listener, "Reducing " + totalRows + " x " + teacherDimension + " to " + requested + " principal components"); - final RandomizedPca.Result pca = RandomizedPca.fitTransform(embeddings, rows, + final RandomizedPca.Result pca = RandomizedPca.fitTransform(embeddings, totalRows, teacherDimension, requested, PCA_SEED); transformed = pca.transformed(); components = requested; explainedVarianceRatio = pca.explainedVarianceRatio(); } - final float[] weights = zipfWeights(rows, SIF_COEFFICIENT); - for (int row = 0; row < rows; row++) { + final float[] weights = zipfWeights(totalRows, SIF_COEFFICIENT); + for (int row = 0; row < totalRows; row++) { final int base = row * components; final float weight = weights[row]; for (int d = 0; d < components; d++) { @@ -221,15 +310,107 @@ public static Result distill(Path teacherDirectory, Path outputDirectory, int pc report(listener, "Writing and verifying the model directory " + outputDirectory); Files.createDirectories(outputDirectory); - SafetensorsWriter.writeMatrix(outputDirectory.resolve(ModelFileNames.SAFETENSORS), rows, + SafetensorsWriter.writeMatrix(outputDirectory.resolve(ModelFileNames.SAFETENSORS), totalRows, components, transformed); tokenizer.writeCleaned(outputDirectory.resolve(ModelFileNames.TOKENIZER_JSON)); Files.writeString(outputDirectory.resolve(ModelFileNames.CONFIG), configJson(teacherDirectory, pcaDims, components)); copySentencePieceModel(teacherDirectory, outputDirectory); + final Path termsFile = outputDirectory.resolve(ModelFileNames.TERMS); + if (termList.isEmpty()) { + // A stale terms file from a previous run would no longer match the matrix's row count. + Files.deleteIfExists(termsFile); + } else { + Files.write(termsFile, termList); + } final ModelAssembler.Result assembled = ModelAssembler.assemble(outputDirectory); - return new Result(assembled.family(), assembled.vocabularySize(), teacherDimension, - assembled.dimension(), explainedVarianceRatio); + return new Result(assembled.family(), assembled.vocabularySize(), assembled.termCount(), + teacherDimension, assembled.dimension(), explainedVarianceRatio); + } + + /** + * Encodes the term rows: each term is segmented by the teacher's own tokenizer, wrapped as a + * full input sequence, and mean-pooled through the teacher, filling the matrix rows after the + * vocabulary rows. Sequences vary in length and a batch must not be ragged, so equal-length + * sequences are batched together. + * + * @param termList The normalized terms, in row order. + * @param tokenizer The teacher's parsed tokenizer. + * @param teacherDirectory The teacher's directory, for the segmenter. + * @param encoder The open teacher encoder. + * @param embeddings The matrix being filled, {@code totalRows * teacherDimension}. + * @param vocabularyRows The number of vocabulary rows preceding the term rows. + * @param teacherDimension The teacher's hidden dimension. + * @param listener Receives one progress line per batch; may be {@code null}. + * @throws IOException Thrown if reading the teacher's SentencePiece file fails. + */ + private static void encodeTerms(List termList, TeacherTokenizer tokenizer, + Path teacherDirectory, OnnxTeacherEncoder encoder, + float[] embeddings, int vocabularyRows, int teacherDimension, + ProgressListener listener) throws IOException { + if (termList.isEmpty()) { + return; + } + report(listener, "Encoding " + termList.size() + + " terms through the teacher's own segmentation"); + final TermSegmenter segmenter = TermSegmenter.forTeacher(tokenizer, teacherDirectory); + final long[][] sequences = new long[termList.size()][]; + for (int t = 0; t < sequences.length; t++) { + sequences[t] = tokenizer.inputSequence(segmenter.pieces(termList.get(t))); + } + final Integer[] byLength = new Integer[sequences.length]; + for (int t = 0; t < byLength.length; t++) { + byLength[t] = t; + } + Arrays.sort(byLength, Comparator.comparingInt(t -> sequences[t].length)); + int encoded = 0; + while (encoded < byLength.length) { + int end = encoded + 1; + while (end < byLength.length && end - encoded < BATCH_SIZE + && sequences[byLength[end]].length == sequences[byLength[encoded]].length) { + end++; + } + final long[][] batch = new long[end - encoded][]; + for (int b = 0; b < batch.length; b++) { + batch[b] = sequences[byLength[encoded + b]]; + } + final float[][] pooled = encoder.encodeBatch(batch); + for (int b = 0; b < batch.length; b++) { + System.arraycopy(pooled[b], 0, embeddings, + (vocabularyRows + byLength[encoded + b]) * teacherDimension, teacherDimension); + } + encoded = end; + report(listener, "Encoded " + encoded + " / " + termList.size() + " terms"); + } + } + + /** + * Normalizes and deduplicates the requested terms before any teacher work: each term becomes + * its lower-cased words joined by single spaces, and terms normalizing to the same form are + * kept once, in first-occurrence order. + * + * @param terms The requested terms. + * @return The normalized, duplicate-free terms. + * @throws IllegalArgumentException Thrown if {@code terms} is {@code null}, contains + * {@code null}, or contains a term with no letter or digit. + */ + private static List prepareTerms(List terms) { + if (terms == null) { + throw new IllegalArgumentException("Terms must not be null"); + } + final Set prepared = new LinkedHashSet<>(terms.size() * 2); + for (final String term : terms) { + if (term == null) { + throw new IllegalArgumentException("Terms must not contain null"); + } + final String normalized = TermTable.normalizeTerm(term); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("Term '" + term + + "' has no letter or digit; it cannot be matched in text"); + } + prepared.add(normalized); + } + return List.copyOf(prepared); } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java index 9139426cb9..657b814fc0 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java @@ -48,6 +48,9 @@ final class ModelFileNames { /** The tokenizer configuration carrying the WordPiece {@code do_lower_case} switch. */ static final String TOKENIZER_CONFIG = "tokenizer_config.json"; + /** The optional term rows of the matrix, one normalized term per line in row order. */ + static final String TERMS = "terms.txt"; + /** The file names SentencePiece models ship their trained {@code .model} under, in try order. */ static final List SENTENCEPIECE_MODELS = List.of("sentencepiece.bpe.model", "spiece.model", "tokenizer.model"); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 12f5377e3e..c521d5049c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -25,6 +25,7 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.function.IntConsumer; import java.util.function.IntPredicate; import opennlp.subword.sentencepiece.SentencePieceTokenizer; @@ -58,6 +59,13 @@ * count of pooled pieces, not the sum of weights. A text with no in-vocabulary pieces yields a * zero vector.

* + *

A model directory may additionally carry a {@code terms.txt}: whole words and multi-word + * phrases distilled through the teacher as units, owning the matrix rows after the subword rows + * (see {@link ModelDistiller}). Embedding then matches the text against these terms greedily + * longest-first, pools a matched term's single row in place of its words' subword pieces, and + * tokenizes only the text between matches; a model without the file embeds exactly as before. + * Term matching is case-insensitive regardless of the subword tokenizer's casing.

+ * *

Instances are immutable and safe for concurrent use after construction.

* *

Warning: Experimental new feature; the API might change in a later release.

@@ -109,12 +117,14 @@ public enum Normalization { // Per-row L2 norms and special-token mask, precomputed at load time for the neighbor scan. private final double[] rowNorms; private final boolean[] specialRows; + // The term rows after the subword rows; empty for a model without a term table. + private final TermTable terms; /** Holds the loaded, validated state; callers reach this through the {@code load} factories. */ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, EmbeddingVocabulary vocabulary, SubwordTokenizer tokenizer, IntPredicate skipPieceId, boolean normalize, double[] rowNorms, - boolean[] specialRows) { + boolean[] specialRows, TermTable terms) { this.embeddings = embeddings; this.weights = weights; this.dimension = dimension; @@ -124,6 +134,7 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, this.normalize = normalize; this.rowNorms = rowNorms; this.specialRows = specialRows; + this.terms = terms; } /** @@ -162,9 +173,13 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException throw new IllegalArgumentException( "Model directory does not exist or is not a directory: " + modelDirectory); } + final Path termsFile = modelDirectory.resolve(ModelFileNames.TERMS); + final List termLines = Files.isRegularFile(termsFile) + ? Files.readAllLines(termsFile) : List.of(); final Path vocabularyFile = modelDirectory.resolve(ModelFileNames.VOCABULARY); if (Files.isRegularFile(vocabularyFile)) { - return loadWordpieceDirectory(modelDirectory, vocabularyFile); + return loadWordpieceDirectory(modelDirectory, vocabularyFile, termLines, + termsFile.toString()); } final Path sentencePieceModelFile = ModelFileNames.firstRegularFile(modelDirectory, ModelFileNames.SENTENCEPIECE_MODELS); @@ -172,7 +187,8 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException if (sentencePieceModelFile != null && Files.isRegularFile(tokenizerJsonFile)) { return loadSentencePiece(sentencePieceModelFile, tokenizerJsonFile, requiredFile(modelDirectory, ModelFileNames.SAFETENSORS), - requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG))); + requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG)), + termLines, termsFile.toString()); } if (Files.isRegularFile(tokenizerJsonFile)) { throw new InvalidFormatException("Model directory " + modelDirectory + " has a " @@ -190,13 +206,17 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException * Loads the WordPiece directory layout, reading the tokenizer and pooling switches from the * model's own configuration files. * - * @param modelDirectory The model directory. - * @param vocabularyFile The directory's {@code vocab.txt}. + * @param modelDirectory The model directory. + * @param vocabularyFile The directory's {@code vocab.txt}. + * @param termLines The directory's terms in row order; empty without a terms file. + * @param termsSourceName The terms' source, for error messages. * @return The loaded model. * @throws IOException Thrown if reading a file fails. */ private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, - Path vocabularyFile) + Path vocabularyFile, + List termLines, + String termsSourceName) throws IOException { final Path safetensorsFile = requiredFile(modelDirectory, ModelFileNames.SAFETENSORS); final Path tokenizerConfigFile = @@ -219,8 +239,8 @@ private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, + "with load(vocabularyFile, safetensorsFile, casing, normalization) after choosing " + "deliberately"); } - return load(vocabularyFile, safetensorsFile, - lowerCase ? Casing.UNCASED : Casing.CASED, normalization); + return loadWordpiece(vocabularyFile, safetensorsFile, + lowerCase ? Casing.UNCASED : Casing.CASED, normalization, termLines, termsSourceName); } /** @@ -298,6 +318,27 @@ private static Path requiredFile(Path modelDirectory, String name) public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFile, Casing casing, Normalization normalization) throws IOException { + return loadWordpiece(vocabularyFile, safetensorsFile, casing, normalization, List.of(), + ModelFileNames.TERMS); + } + + /** + * Loads the WordPiece layout with an optional term table. + * + * @param vocabularyFile The {@code vocab.txt} file. + * @param safetensorsFile The {@code model.safetensors} file. + * @param casing The tokenizer's casing. + * @param normalization The pooling normalization. + * @param termLines The terms in row order; empty without a term table. + * @param termsSourceName The terms' source, for error messages. + * @return The loaded model. + * @throws IOException Thrown if reading a file fails. + */ + private static StaticEmbeddingModel loadWordpiece(Path vocabularyFile, Path safetensorsFile, + Casing casing, Normalization normalization, + List termLines, + String termsSourceName) + throws IOException { if (vocabularyFile == null) { throw new IllegalArgumentException("VocabularyFile must not be null"); } @@ -311,7 +352,9 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil throw new IllegalArgumentException("Normalization must not be null"); } final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromVocabTxt(vocabularyFile); - final Matrix matrix = readMatrix(vocabulary, safetensorsFile, vocabularyFile.toString()); + final TermTable terms = TermTable.of(termLines, vocabulary.size(), termsSourceName); + final Matrix matrix = readMatrix(vocabulary, terms.size(), safetensorsFile, + vocabularyFile.toString()); final int unknownId = vocabulary.id(WordpieceTokenizer.BERT_UNK_TOKEN); if (unknownId < 0) { throw new InvalidFormatException("Vocabulary " + vocabularyFile + " has no " @@ -328,8 +371,9 @@ public static StaticEmbeddingModel load(Path vocabularyFile, Path safetensorsFil id -> id == unknownId || id == classificationId || id == separatorId; return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, - rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size()), - specialRows(vocabulary, WORDPIECE_SPECIAL_TOKENS)); + rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size() + terms.size()), + specialRows(vocabulary, WORDPIECE_SPECIAL_TOKENS, vocabulary.size() + terms.size()), + terms); } /** @@ -402,6 +446,29 @@ public static StaticEmbeddingModel loadSentencePiece(Path sentencePieceModelFile Path safetensorsFile, Normalization normalization) throws IOException { + return loadSentencePiece(sentencePieceModelFile, tokenizerJsonFile, safetensorsFile, + normalization, List.of(), ModelFileNames.TERMS); + } + + /** + * Loads the SentencePiece layout with an optional term table. + * + * @param sentencePieceModelFile The trained SentencePiece {@code .model} file. + * @param tokenizerJsonFile The Unigram {@code tokenizer.json} file. + * @param safetensorsFile The {@code model.safetensors} file. + * @param normalization The pooling normalization. + * @param termLines The terms in row order; empty without a term table. + * @param termsSourceName The terms' source, for error messages. + * @return The loaded model. + * @throws IOException Thrown if reading a file fails. + */ + private static StaticEmbeddingModel loadSentencePiece(Path sentencePieceModelFile, + Path tokenizerJsonFile, + Path safetensorsFile, + Normalization normalization, + List termLines, + String termsSourceName) + throws IOException { if (sentencePieceModelFile == null) { throw new IllegalArgumentException("SentencePieceModelFile must not be null"); } @@ -416,16 +483,20 @@ public static StaticEmbeddingModel loadSentencePiece(Path sentencePieceModelFile } final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromTokenizerJson(tokenizerJsonFile); + final TermTable terms = TermTable.of(termLines, vocabulary.size(), termsSourceName); final SentencePieceTokenizer tokenizer = SentencePieceTokenizer.load(sentencePieceModelFile); requireVocabularyCoverage(tokenizer, vocabulary, sentencePieceModelFile, tokenizerJsonFile); - final Matrix matrix = readMatrix(vocabulary, safetensorsFile, tokenizerJsonFile.toString()); + final Matrix matrix = readMatrix(vocabulary, terms.size(), safetensorsFile, + tokenizerJsonFile.toString()); final IntPredicate skipPieceId = id -> tokenizer.isUnknown(id) || tokenizer.isControl(id); return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, - rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size()), - specialRows(vocabulary, SENTENCEPIECE_SPECIAL_TOKENS)); + rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size() + terms.size()), + specialRows(vocabulary, SENTENCEPIECE_SPECIAL_TOKENS, + vocabulary.size() + terms.size()), + terms); } /** @@ -473,24 +544,29 @@ private record Matrix(float[] embeddings, float[] weights, int dimension) { /** * Reads the embedding matrix and the optional {@code weights} tensor, holding both to the - * vocabulary's size. + * model's row count: the vocabulary's size plus the term count. * * @param vocabulary The matrix row vocabulary. + * @param termCount The number of term rows after the vocabulary rows. * @param safetensorsFile The safetensors file to read. * @param vocabularySourceName The vocabulary's source, for error messages. * @return The matrix, its optional weights, and its dimension. * @throws InvalidFormatException Thrown if the matrix's row count or the weights tensor's - * length disagrees with the vocabulary size, or the matrix contains a non-finite value. + * length disagrees with the model's row count, or the matrix contains a non-finite value. * @throws IOException Thrown if reading the file fails. */ - private static Matrix readMatrix(EmbeddingVocabulary vocabulary, Path safetensorsFile, - String vocabularySourceName) throws IOException { + private static Matrix readMatrix(EmbeddingVocabulary vocabulary, int termCount, + Path safetensorsFile, String vocabularySourceName) + throws IOException { + final int expectedRows = vocabulary.size() + termCount; final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); final String matrixName = tensors.singleMatrixTensorName(); final TensorInfo matrixInfo = tensors.tensorInfo(matrixName); - if (matrixInfo.shape()[0] != vocabulary.size()) { + if (matrixInfo.shape()[0] != expectedRows) { throw new InvalidFormatException("Vocabulary " + vocabularySourceName + " has " - + vocabulary.size() + " tokens but embedding matrix '" + matrixName + "' in " + + vocabulary.size() + " tokens" + + (termCount > 0 ? " plus " + termCount + " terms" : "") + + " but embedding matrix '" + matrixName + "' in " + safetensorsFile + " has " + matrixInfo.shape()[0] + " rows; these files do not " + "belong to the same model"); } @@ -510,10 +586,10 @@ private static Matrix readMatrix(EmbeddingVocabulary vocabulary, Path safetensor float[] weights = null; if (tensors.tensorNames().contains(WEIGHTS_TENSOR_NAME)) { weights = tensors.readFloats(WEIGHTS_TENSOR_NAME); - if (weights.length != vocabulary.size()) { + if (weights.length != expectedRows) { throw new InvalidFormatException("Tensor '" + WEIGHTS_TENSOR_NAME + "' in " - + safetensorsFile + " has " + weights.length + " elements but the vocabulary has " - + vocabulary.size() + " tokens"); + + safetensorsFile + " has " + weights.length + " elements but the model has " + + expectedRows + " rows"); } } return new Matrix(embeddings, weights, dimension); @@ -541,15 +617,17 @@ private static double[] rowNorms(float[] embeddings, int dimension, int rowCount } /** - * {@return the mask of rows holding special tokens, excluded from neighbor results} + * {@return the mask of rows holding special tokens, excluded from neighbor results; term rows + * are never special} * * @param vocabulary The matrix row vocabulary. * @param specialTokens The special-token strings of the model's convention; tokens absent * from the vocabulary are simply not marked. + * @param totalRows The model's row count, the vocabulary's size plus the term count. */ private static boolean[] specialRows(EmbeddingVocabulary vocabulary, - Set specialTokens) { - final boolean[] specialRows = new boolean[vocabulary.size()]; + Set specialTokens, int totalRows) { + final boolean[] specialRows = new boolean[totalRows]; for (final String special : specialTokens) { final int row = vocabulary.id(special); if (row >= 0) { @@ -586,20 +664,10 @@ public float[] embed(String text) { if (text == null) { throw new IllegalArgumentException("Text must not be null"); } - final List pieces = tokenizer.encode(text); final float[] sum = new float[dimension]; - int pooledCount = 0; - for (int i = 0; i < pieces.size(); i++) { - final SubwordPiece piece = pieces.get(i); - if (skipPieceId.test(piece.id())) { - continue; - } - final int row = vocabulary.id(piece.piece()); - if (row < 0) { - throw new IllegalStateException("Tokenizer produced piece '" + piece.piece() - + "' that has no matrix row; load-time validation admits no such piece, so this " - + "indicates a construction bug, not an input problem"); - } + // The count travels through the IntConsumer as a one-element array. + final int[] pooled = new int[1]; + forEachPooledRow(text, row -> { final int base = row * dimension; if (weights == null) { for (int d = 0; d < dimension; d++) { @@ -611,9 +679,9 @@ public float[] embed(String text) { sum[d] += embeddings[base + d] * weight; } } - pooledCount++; - } - final int denominator = Math.max(pooledCount, 1); + pooled[0]++; + }); + final int denominator = Math.max(pooled[0], 1); for (int d = 0; d < dimension; d++) { sum[d] /= denominator; } @@ -630,17 +698,74 @@ public float[] embed(String text) { return sum; } + /** + * Feeds every matrix row a text pools to the action, in text order: matched terms' rows where + * the term table matches, and subword piece rows everywhere else. Without a term table this + * is exactly the piece walk over the whole text. + * + * @param text The text to fold into rows. + * @param action Receives each pooled row. + */ + private void forEachPooledRow(String text, IntConsumer action) { + if (terms.size() == 0) { + forEachPieceRow(text, action); + return; + } + int cursor = 0; + for (final TermTable.Match match : terms.matches(text)) { + if (match.start() > cursor) { + forEachPieceRow(text.substring(cursor, match.start()), action); + } + action.accept(match.row()); + cursor = match.end(); + } + if (cursor < text.length()) { + forEachPieceRow(text.substring(cursor), action); + } + } + + /** + * Feeds the matrix row of every poolable subword piece of a text to the action. + * + * @param text The text to tokenize. + * @param action Receives each piece's row. + */ + private void forEachPieceRow(String text, IntConsumer action) { + final List pieces = tokenizer.encode(text); + for (int i = 0; i < pieces.size(); i++) { + final SubwordPiece piece = pieces.get(i); + if (skipPieceId.test(piece.id())) { + continue; + } + final int row = vocabulary.id(piece.piece()); + if (row < 0) { + throw new IllegalStateException("Tokenizer produced piece '" + piece.piece() + + "' that has no matrix row; load-time validation admits no such piece, so this " + + "indicates a construction bug, not an input problem"); + } + action.accept(row); + } + } + /** {@inheritDoc} */ @Override public int dimension() { return dimension; } - /** {@return the number of tokens in this model's vocabulary} */ + /** {@return the number of subword tokens in this model's vocabulary, without term rows} */ public int vocabularySize() { return vocabulary.size(); } + /** + * {@return the number of term rows appended after the subword vocabulary, {@code 0} for a + * model without a term table} + */ + public int termCount() { + return terms.size(); + } + /** * Cosine similarity between two pieces of text's pooled embeddings. * @@ -662,7 +787,8 @@ public double similarity(String text1, String text2) { /** * Finds the vocabulary tokens whose vectors are nearest a piece of text's pooled embedding, - * most similar first. This is a brute-force scan over the whole vocabulary. + * most similar first. This is a brute-force scan over the whole table; a model with a term + * table returns matching terms as neighbors like any token. * * @param text The query text. Must not be {@code null}. * @param topK The maximum number of results. Must be at least 1. @@ -736,18 +862,10 @@ private void requirePositive(int topK) { * * @param terms The terms to fold and exclude. */ - private int[] excludedRows(String... terms) { + private int[] excludedRows(String... queryTerms) { final SortedSet rows = new TreeSet<>(); - for (final String term : terms) { - for (final SubwordPiece piece : tokenizer.encode(term)) { - if (skipPieceId.test(piece.id())) { - continue; - } - final int row = vocabulary.id(piece.piece()); - if (row >= 0) { - rows.add(row); - } - } + for (final String queryTerm : queryTerms) { + forEachPooledRow(queryTerm, rows::add); } final int[] sorted = new int[rows.size()]; int i = 0; @@ -812,12 +930,22 @@ private List nearestNeighbors(float[] query, int topK, int[] sortedExc } final Neighbor[] ordered = new Neighbor[best.size()]; for (int i = ordered.length - 1; i >= 0; i--) { - ordered[i] = new Neighbor(vocabulary.token(best.minRow()), best.minSimilarity()); + ordered[i] = new Neighbor(rowToken(best.minRow()), best.minSimilarity()); best.removeMin(); } return List.of(ordered); } + /** + * {@return the string of a matrix row: the vocabulary token of a subword row, the term of a + * term row} + * + * @param row The matrix row. + */ + private String rowToken(int row) { + return row < vocabulary.size() ? vocabulary.token(row) : terms.term(row); + } + /** * {@return the cosine similarity of two vectors, or {@code 0} when either has no direction} * diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java index a9b5c1584e..f10abce892 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TeacherTokenizer.java @@ -20,13 +20,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Pattern; import opennlp.tools.util.InvalidFormatException; @@ -53,11 +53,8 @@ */ final class TeacherTokenizer { - /** Model2Vec's default token removal pattern; matched from the start, like Python re.match. */ - private static final Pattern UNUSED_TOKEN_PATTERN = Pattern.compile("\\[unused\\d+\\]"); - - /** Separates the items of a string post-processor template such as {@code "[CLS] $A [SEP]"}. */ - private static final Pattern TEMPLATE_ITEM_SEPARATOR = Pattern.compile("\\s+"); + /** The prefix of the BERT-style placeholder tokens Model2Vec's cleaning drops. */ + private static final String UNUSED_TOKEN_PREFIX = "[unused"; /** Marks a template item as the sequence placeholder rather than a special token. */ private static final String SEQUENCE_PLACEHOLDER_PREFIX = "$"; @@ -79,16 +76,20 @@ final class TeacherTokenizer { private final int padTokenId; private final int[] bosIds; private final int[] eosIds; + private final Map idByOriginalToken; + private final Boolean lowerCase; /** Holds the parsed state; built by {@link #read(Path, Path)}. */ private TeacherTokenizer(String json, String inputName, String modelType, - List tokensByOriginalId, int[] keptOriginalIds, + List tokensByOriginalId, + Map idByOriginalToken, int[] keptOriginalIds, int originalUnkId, String unkToken, String padToken, int padTokenId, - int[] bosIds, int[] eosIds) { + int[] bosIds, int[] eosIds, Boolean lowerCase) { this.json = json; this.inputName = inputName; this.modelType = modelType; this.tokensByOriginalId = tokensByOriginalId; + this.idByOriginalToken = idByOriginalToken; this.keptOriginalIds = keptOriginalIds; this.originalUnkId = originalUnkId; this.unkToken = unkToken; @@ -96,6 +97,7 @@ private TeacherTokenizer(String json, String inputName, String modelType, this.padTokenId = padTokenId; this.bosIds = bosIds; this.eosIds = eosIds; + this.lowerCase = lowerCase; } /** @@ -134,6 +136,7 @@ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) List tokensById = null; String unkToken = null; Long unkId = null; + Boolean lowerCase = null; Set addedContents = Set.of(); PostProcessor postProcessor = new PostProcessor(List.of(), List.of(), null, null, Map.of()); if (cursor.peek() == '}') { @@ -155,6 +158,7 @@ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) } case "added_tokens" -> addedContents = parseAddedTokenContents(cursor); case "post_processor" -> postProcessor = parsePostProcessor(cursor); + case "normalizer" -> lowerCase = parseNormalizerLowercase(cursor); default -> cursor.skipValue(); } cursor.skipWhitespace(); @@ -215,7 +219,7 @@ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) final List kept = new ArrayList<>(tokensById.size()); for (int id = 0; id < tokensById.size(); id++) { final String token = tokensById.get(id); - if (UNUSED_TOKEN_PATTERN.matcher(token).lookingAt()) { + if (isUnusedToken(token)) { continue; } if (addedContents.contains(token) && !keepSpecial.contains(token)) { @@ -223,9 +227,81 @@ static TeacherTokenizer read(Path tokenizerJsonFile, Path tokenizerConfigFile) } kept.add(id); } - return new TeacherTokenizer(json, inputName, modelType, tokensById, + return new TeacherTokenizer(json, inputName, modelType, tokensById, idByToken, kept.stream().mapToInt(Integer::intValue).toArray(), originalUnkId, unkToken, padToken, - padTokenId, bosIds, eosIds); + padTokenId, bosIds, eosIds, lowerCase); + } + + /** + * Reads the flat {@code lowercase} boolean of a {@code normalizer} object, for the BERT + * normalizer a WordPiece tokenizer carries. Shared with {@link ModelAssembler}, which derives + * a distilled directory's {@code do_lower_case} from the same flag. + * + * @param cursor The cursor, positioned at the normalizer value. + * @return The {@code lowercase} flag, or {@code null} when the value is JSON null or the flag + * is absent (for example a nested normalizer with no flat flag). + * @throws InvalidFormatException Thrown if the normalizer object is malformed. + */ + static Boolean parseNormalizerLowercase(JsonCursor cursor) throws InvalidFormatException { + if (cursor.peek() != '{') { + cursor.skipValue(); + return null; + } + cursor.expect('{'); + cursor.skipWhitespace(); + Boolean lowerCase = null; + if (cursor.peek() == '}') { + cursor.consume(); + return null; + } + while (true) { + cursor.skipWhitespace(); + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + if ("lowercase".equals(key)) { + if (cursor.consumeLiteral("true")) { + lowerCase = Boolean.TRUE; + } else if (cursor.consumeLiteral("false")) { + lowerCase = Boolean.FALSE; + } else { + cursor.skipValue(); + } + } else { + cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + continue; + } + if (next == '}') { + return lowerCase; + } + throw cursor.malformed("Expected ',' or '}' after a normalizer field, got '" + next + "'"); + } + } + + /** + * {@return whether a token starts with a BERT-style unused placeholder, {@code [unused} + * followed by at least one ASCII digit and {@code ]}} + * + *

Model2Vec's cleaning drops these tokens by a prefix match, so a longer token starting + * with the placeholder form is dropped the same way.

+ * + * @param token The vocabulary token. + */ + private static boolean isUnusedToken(String token) { + if (!token.startsWith(UNUSED_TOKEN_PREFIX)) { + return false; + } + int i = UNUSED_TOKEN_PREFIX.length(); + final int digitsStart = i; + while (i < token.length() && token.charAt(i) >= '0' && token.charAt(i) <= '9') { + i++; + } + return i > digitsStart && i < token.length() && token.charAt(i) == ']'; } /** @@ -307,6 +383,58 @@ long[] inputSequence(int row) { return sequence; } + /** + * The teacher input sequence of a segmented term: the begin-of-sequence ids, each piece's + * original id (the unknown token's id for a piece the vocabulary does not carry), and the + * end-of-sequence ids. + * + * @param pieces The term's piece strings, as the teacher's own segmenter produced them. Must + * not be {@code null}. + * @return The teacher input ids. + * @throws IllegalArgumentException Thrown if {@code pieces} is {@code null}. + */ + long[] inputSequence(List pieces) { + if (pieces == null) { + throw new IllegalArgumentException("Pieces must not be null"); + } + final long[] sequence = new long[bosIds.length + pieces.size() + eosIds.length]; + int i = 0; + for (final int id : bosIds) { + sequence[i++] = id; + } + for (final String piece : pieces) { + final Integer id = idByOriginalToken.get(piece); + sequence[i++] = id == null ? originalUnkId : id; + } + for (final int id : eosIds) { + sequence[i++] = id; + } + return sequence; + } + + /** + * Looks up the token string of a matrix row. + * + * @param row The matrix row, within {@code [0, vocabularySize())}. + * @return The surviving token at that row. + */ + String rowToken(int row) { + return tokensByOriginalId.get(keptOriginalIds[row]); + } + + /** {@return the whole vocabulary in the teacher's id order, for an id-is-index segmenter} */ + List tokensByOriginalId() { + return Collections.unmodifiableList(tokensByOriginalId); + } + + /** + * {@return the {@code normalizer.lowercase} flag of the teacher's {@code tokenizer.json}, or + * {@code null} when the tokenizer does not state it} + */ + Boolean lowerCase() { + return lowerCase; + } + /** * Writes the cleaned {@code tokenizer.json}: the surviving vocabulary renumbered, the * added-token overlay pruned to the unknown and pad tokens, the post-processor nulled, and @@ -875,12 +1003,22 @@ private static List> parseTemplate(JsonCursor cursor) final List bos = new ArrayList<>(1); final List eos = new ArrayList<>(1); if (cursor.peek() == '"') { + // The template is items separated by whitespace runs, such as "[CLS] $A [SEP]". final String template = cursor.parseString(); List current = bos; - for (final String part : TEMPLATE_ITEM_SEPARATOR.split(template)) { - if (part.isEmpty()) { + final int length = template.length(); + int i = 0; + while (i < length) { + final int c = template.codePointAt(i); + if (Character.isWhitespace(c)) { + i += Character.charCount(c); continue; } + final int start = i; + while (i < length && !Character.isWhitespace(template.codePointAt(i))) { + i += Character.charCount(template.codePointAt(i)); + } + final String part = template.substring(start, i); if (part.startsWith(SEQUENCE_PLACEHOLDER_PREFIX)) { current = eos; } else { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermSegmenter.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermSegmenter.java new file mode 100644 index 0000000000..5b3979abaf --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermSegmenter.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.function.IntPredicate; + +import opennlp.subword.sentencepiece.SentencePieceTokenizer; +import opennlp.tools.tokenize.SubwordPiece; +import opennlp.tools.tokenize.SubwordTokenizer; +import opennlp.tools.tokenize.WordpieceEncoder; +import opennlp.tools.tokenize.WordpieceTokenizer; + +/** + * Segments a term's text into the piece strings the teacher's own tokenizer would produce, so a + * distillation can run a whole word or phrase through the teacher the way the teacher would see + * it in running text. A WordPiece teacher segments through a {@link WordpieceEncoder} built over + * the teacher's full vocabulary; a Unigram teacher segments through its trained SentencePiece + * {@code .model} file. + * + *

The sequence-delimiter pieces the segmenter itself wraps around an encoding are removed; + * {@link TeacherTokenizer#inputSequence(List)} adds the teacher's own wrapping when the pieces + * are turned into an input sequence.

+ */ +final class TermSegmenter { + + private final SubwordTokenizer tokenizer; + private final Set dropPieces; + private final IntPredicate dropPieceId; + + /** Holds the segmenter and its piece filters; built by {@link #forTeacher}. */ + private TermSegmenter(SubwordTokenizer tokenizer, Set dropPieces, + IntPredicate dropPieceId) { + this.tokenizer = tokenizer; + this.dropPieces = dropPieces; + this.dropPieceId = dropPieceId; + } + + /** + * Builds the segmenter matching a teacher's tokenizer family. + * + * @param teacher The teacher's parsed tokenizer. Must not be {@code null}. + * @param teacherDirectory The teacher's directory, holding the trained SentencePiece + * {@code .model} file when the teacher is a Unigram model. Must not be + * {@code null}. + * @return The segmenter. + * @throws IllegalArgumentException Thrown if an argument is {@code null}, a Unigram teacher + * has no trained SentencePiece file, or a WordPiece teacher's vocabulary lacks the BERT + * special tokens the encoder wraps with. + * @throws IOException Thrown if reading the SentencePiece file fails. + */ + static TermSegmenter forTeacher(TeacherTokenizer teacher, Path teacherDirectory) + throws IOException { + if (teacher == null) { + throw new IllegalArgumentException("Teacher must not be null"); + } + if (teacherDirectory == null) { + throw new IllegalArgumentException("TeacherDirectory must not be null"); + } + if (TeacherTokenizer.WORDPIECE.equals(teacher.modelType())) { + // The lowercase default matches ModelAssembler's: absent means the uncased convention. + final boolean lowerCase = teacher.lowerCase() == null || teacher.lowerCase(); + final WordpieceEncoder encoder; + try { + encoder = new WordpieceEncoder(teacher.tokensByOriginalId(), lowerCase, + WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN, + teacher.unkToken()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("The teacher's WordPiece vocabulary cannot segment " + + "terms: " + e.getMessage(), e); + } + return new TermSegmenter(encoder, + Set.of(WordpieceTokenizer.BERT_CLS_TOKEN, WordpieceTokenizer.BERT_SEP_TOKEN), + id -> false); + } + final Path sentencePieceModelFile = ModelFileNames.firstRegularFile(teacherDirectory, + ModelFileNames.SENTENCEPIECE_MODELS); + if (sentencePieceModelFile == null) { + throw new IllegalArgumentException("Teacher directory " + teacherDirectory + " has no " + + "trained SentencePiece file (one of " + + String.join(", ", ModelFileNames.SENTENCEPIECE_MODELS) + "); distilling terms " + + "needs the teacher's own segmentation"); + } + final SentencePieceTokenizer sentencePiece = + SentencePieceTokenizer.load(sentencePieceModelFile); + return new TermSegmenter(sentencePiece, Set.of(), + id -> id >= 0 && sentencePiece.isControl(id)); + } + + /** + * Segments a term into the teacher's piece strings, without sequence delimiters. + * + * @param term The term text. Must not be {@code null}. + * @return The piece strings in order. + * @throws IllegalArgumentException Thrown if {@code term} is {@code null}. + */ + List pieces(String term) { + if (term == null) { + throw new IllegalArgumentException("Term must not be null"); + } + final List encoded = tokenizer.encode(term); + final List pieces = new ArrayList<>(encoded.size()); + for (final SubwordPiece piece : encoded) { + if (dropPieces.contains(piece.piece()) || dropPieceId.test(piece.id())) { + continue; + } + pieces.add(piece.piece()); + } + return pieces; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermTable.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermTable.java new file mode 100644 index 0000000000..80555a96db --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/TermTable.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.util.StringUtil; + +/** + * The term rows of a static embedding matrix: whole words and multi-word phrases that were + * distilled through the teacher as units and sit after the subword rows. Matching a text against + * the table finds the greedily longest term at each word position, so "writ of habeas corpus" + * wins over "habeas corpus" wins over the subword pieces of each word. + * + *

A term is stored in normalized form: the lower-cased letter-or-digit word runs of its text, + * joined by single spaces (see {@link #normalizeTerm(String)}). Matching folds each word run of + * the input text the same way, so "Habeas Corpus" and "habeas-corpus" both match the term + * "habeas corpus". The fold is {@link StringUtil#toLowerCase(CharSequence)}, locale-independent + * and one code point to one code point, so word-run boundaries are the same before and after + * folding.

+ * + *

Immutable and safe for concurrent reads after construction.

+ */ +@ThreadSafe +final class TermTable { + + private final List termsByOffset; + private final Map rowByTerm; + private final int firstRow; + private final int maxTermWords; + + /** Holds the validated term-to-row views; built by {@link #of(List, int, String)}. */ + private TermTable(List termsByOffset, Map rowByTerm, int firstRow, + int maxTermWords) { + this.termsByOffset = termsByOffset; + this.rowByTerm = rowByTerm; + this.firstRow = firstRow; + this.maxTermWords = maxTermWords; + } + + /** + * Builds a term table from terms in matrix row order. + * + * @param terms The terms; the term at index {@code i} owns matrix row + * {@code firstRow + i}. Every term must already be in its normalized form. + * Must not be {@code null}. + * @param firstRow The matrix row of the first term, the number of subword rows. + * @param sourceName The terms' source, for error messages. + * @return The table. + * @throws IllegalArgumentException Thrown if {@code terms} is {@code null}. + * @throws InvalidFormatException Thrown if a term is {@code null}, not in normalized form, or + * appears more than once. + */ + static TermTable of(List terms, int firstRow, String sourceName) + throws InvalidFormatException { + if (terms == null) { + throw new IllegalArgumentException("Terms must not be null"); + } + final Map rowByTerm = new HashMap<>(terms.size() * 2); + int maxTermWords = 0; + for (int i = 0; i < terms.size(); i++) { + final String term = terms.get(i); + if (term == null || !term.equals(normalizeTerm(term)) || term.isEmpty()) { + throw new InvalidFormatException("Term " + i + " in " + sourceName + " ('" + term + + "') is not in normalized form (lower-cased words joined by single spaces)"); + } + if (rowByTerm.putIfAbsent(term, firstRow + i) != null) { + throw new InvalidFormatException("Term '" + term + "' appears more than once in " + + sourceName); + } + maxTermWords = Math.max(maxTermWords, countWords(term)); + } + return new TermTable(List.copyOf(terms), Map.copyOf(rowByTerm), firstRow, maxTermWords); + } + + /** + * {@return a term's normalized form: its lower-cased letter-or-digit word runs joined by + * single spaces, or the empty string when the text contains no such run} + * + * @param text The term text. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + static String normalizeTerm(String text) { + if (text == null) { + throw new IllegalArgumentException("Text must not be null"); + } + final StringBuilder normalized = new StringBuilder(text.length()); + final String folded = StringUtil.toLowerCase(text); + final int length = folded.length(); + int i = 0; + while (i < length) { + final int c = folded.codePointAt(i); + if (Character.isLetterOrDigit(c)) { + if (normalized.length() > 0) { + normalized.append(' '); + } + while (i < length && Character.isLetterOrDigit(folded.codePointAt(i))) { + normalized.appendCodePoint(folded.codePointAt(i)); + i += Character.charCount(folded.codePointAt(i)); + } + } else { + i += Character.charCount(c); + } + } + return normalized.toString(); + } + + /** {@return the number of space-separated words of a normalized term} */ + private static int countWords(String term) { + int words = 1; + for (int i = 0; i < term.length(); i++) { + if (term.charAt(i) == ' ') { + words++; + } + } + return words; + } + + /** {@return the number of terms in this table} */ + int size() { + return termsByOffset.size(); + } + + /** + * Looks up the term owning a matrix row. + * + * @param row The matrix row. Must be within {@code [firstRow, firstRow + size())}. + * @return The term at that row. + * @throws IllegalArgumentException Thrown if {@code row} is outside the term rows. + */ + String term(int row) { + final int offset = row - firstRow; + if (offset < 0 || offset >= termsByOffset.size()) { + throw new IllegalArgumentException("Row " + row + " is outside the term rows [" + + firstRow + ", " + (firstRow + termsByOffset.size()) + ")"); + } + return termsByOffset.get(offset); + } + + /** + * A term match in a text: the term's matrix row and the character range it consumed, from the + * start of its first word to the end of its last. + * + * @param row The matched term's matrix row. + * @param start The inclusive start of the consumed range. + * @param end The exclusive end of the consumed range. + */ + record Match(int row, int start, int end) { + } + + /** + * Finds every term of this table in a text, greedily longest-first: at each word, the longest + * matching term consumes its words, and matching continues after them. Matched ranges never + * overlap and appear in text order. + * + * @param text The text to match. Must not be {@code null}. + * @return The matches in text order; empty when the table is empty or nothing matches. + * @throws IllegalArgumentException Thrown if {@code text} is {@code null}. + */ + List matches(String text) { + if (text == null) { + throw new IllegalArgumentException("Text must not be null"); + } + if (termsByOffset.isEmpty()) { + return List.of(); + } + final List runs = wordRuns(text); + final List matches = new ArrayList<>(); + int i = 0; + while (i < runs.size()) { + int consumed = 0; + for (int n = Math.min(maxTermWords, runs.size() - i); n >= 1; n--) { + final Integer row = rowByTerm.get(joined(runs, i, n)); + if (row != null) { + matches.add(new Match(row, runs.get(i).start(), runs.get(i + n - 1).end())); + consumed = n; + break; + } + } + i += Math.max(consumed, 1); + } + return matches; + } + + /** A word run of the matched text: its character range and its case-folded form. */ + private record Run(int start, int end, String folded) { + } + + /** + * {@return the letter-or-digit word runs of a text, each with its character range and its + * case-folded form} + * + * @param text The text to scan. + */ + private static List wordRuns(String text) { + final List runs = new ArrayList<>(); + final int length = text.length(); + int i = 0; + while (i < length) { + final int c = text.codePointAt(i); + if (Character.isLetterOrDigit(c)) { + final int start = i; + while (i < length && Character.isLetterOrDigit(text.codePointAt(i))) { + i += Character.charCount(text.codePointAt(i)); + } + runs.add(new Run(start, i, StringUtil.toLowerCase(text.substring(start, i)))); + } else { + i += Character.charCount(c); + } + } + return runs; + } + + /** + * {@return the folded forms of {@code n} runs from {@code first}, joined by single spaces, the + * lookup key of a candidate term} + * + * @param runs The text's word runs. + * @param first The first run of the candidate. + * @param n The number of runs of the candidate. + */ + private static String joined(List runs, int first, int n) { + if (n == 1) { + return runs.get(first).folded(); + } + final StringBuilder key = new StringBuilder(); + for (int i = 0; i < n; i++) { + if (i > 0) { + key.append(' '); + } + key.append(runs.get(first + i).folded()); + } + return key.toString(); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java index c08fdbd225..d819adc6d2 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java @@ -74,6 +74,8 @@ public void run(String[] args) { System.out.println("Wrote tokenizer_config.json derived from tokenizer.json"); } System.out.println("Assembled and verified a " + result.family() + " model: " - + result.vocabularySize() + " rows, dimension " + result.dimension()); + + result.vocabularySize() + " rows" + + (result.termCount() > 0 ? " plus " + result.termCount() + " terms" : "") + + ", dimension " + result.dimension()); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java index 56d74f545d..0f7100ca8d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelParams.java @@ -47,4 +47,14 @@ interface DistillModelParams { @ParameterDescription(valueName = "num", description = "The number of principal components to keep, default is 256.") Integer getPcaDims(); + + /** + * {@return the term file to distill as extra rows, or {@code null} for none} + */ + @OptionalParameter + @ParameterDescription(valueName = "file", + description = "A term file: one term per line, text after a tab ignored, so a learned " + + "vocabulary TSV works as-is. Each term is encoded through the teacher as a unit and " + + "added as an extra row, matched greedily longest-first before subword tokenization.") + String getTerms(); } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java index 36f0a71b27..a30f21a498 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/DistillModelTool.java @@ -17,7 +17,10 @@ package opennlp.embeddings.cmdline; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import opennlp.embeddings.ModelDistiller; import opennlp.tools.cmdline.BasicCmdLineTool; @@ -60,8 +63,10 @@ public void run(String[] args) { final ModelDistiller.ProgressListener listener = System.out::println; final ModelDistiller.Result result; try { + final List terms = params.getTerms() == null + ? List.of() : readTerms(Path.of(params.getTerms())); result = ModelDistiller.distill(params.getTeacher(), Path.of(params.getOut()), - params.getPcaDims(), listener); + params.getPcaDims(), terms, listener); } catch (IllegalArgumentException | InvalidFormatException e) { throw new TerminateToolException(1, e.getMessage(), e); } catch (IOException e) { @@ -69,8 +74,30 @@ public void run(String[] args) { "IO error while distilling: " + e.getMessage(), e); } System.out.println("Distilled and verified a " + result.family() + " model: " - + result.vocabularySize() + " rows, " + result.teacherDimension() + "d -> " + + result.vocabularySize() + " rows" + + (result.termCount() > 0 ? " plus " + result.termCount() + " terms" : "") + + ", " + result.teacherDimension() + "d -> " + result.dimension() + "d, PCA kept " + String.format("%.1f", result.explainedVarianceRatio() * 100) + "% of the variance"); } + + /** + * Reads a term file: one term per line, text after the first tab ignored, blank lines + * skipped. A learned vocabulary TSV (term, count, source) therefore works unchanged. + * + * @param file The term file. + * @return The terms in file order. + * @throws IOException Thrown if reading the file fails. + */ + private List readTerms(Path file) throws IOException { + final List terms = new ArrayList<>(); + for (final String line : Files.readAllLines(file)) { + final int tab = line.indexOf('\t'); + final String term = (tab < 0 ? line : line.substring(0, tab)).strip(); + if (!term.isEmpty()) { + terms.add(term); + } + } + return terms; + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java index 8a3804b320..741e4f30db 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/EmbeddingTestFixtures.java @@ -120,6 +120,20 @@ private static void writeVocabularyAndMatrix(Path dir) throws IOException { * @throws IOException Thrown if reading the fixture resource or writing a file fails. */ static void writeSentencePieceDirectory(Path dir) throws IOException { + writeSentencePieceDirectory(dir, List.of()); + } + + /** + * Writes the SentencePiece model directory of {@link #writeSentencePieceDirectory(Path)} with + * additional term rows: the terms land in {@code terms.txt} and the matrix grows one row per + * term, keeping the deterministic {@code row + d * 0.25} cell formula, so a test can predict a + * term row's vector from the model's vocabulary size. + * + * @param dir The directory to write the model files into. + * @param terms The terms in row order; empty for none. + * @throws IOException Thrown if reading the fixture resource or writing a file fails. + */ + static void writeSentencePieceDirectory(Path dir, List terms) throws IOException { final byte[] modelBytes; try (InputStream in = EmbeddingTestFixtures.class.getResourceAsStream(TINY_UNIGRAM_RESOURCE)) { @@ -144,7 +158,10 @@ static void writeSentencePieceDirectory(Path dir) throws IOException { json.append('[').append(jsonString(rows.get(i))).append(",-1.5]"); } Files.writeString(dir.resolve("tokenizer.json"), json.append("]}}").toString()); - final float[][] matrix = new float[rows.size()][SENTENCEPIECE_DIMENSION]; + if (!terms.isEmpty()) { + Files.write(dir.resolve("terms.txt"), terms); + } + final float[][] matrix = new float[rows.size() + terms.size()][SENTENCEPIECE_DIMENSION]; for (int row = 0; row < matrix.length; row++) { for (int d = 0; d < SENTENCEPIECE_DIMENSION; d++) { matrix[row][d] = row + d * 0.25f; diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java index 7ab1855d5c..0d4a82b233 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelDistillerTest.java @@ -19,6 +19,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -167,4 +169,29 @@ void testRejectsABadOutputBeforeResolvingAHubTeacher(String teacher, @TempDir Pa assertThrows(IllegalArgumentException.class, () -> ModelDistiller.distill(teacher, dir.resolve("out"), 0, null)).getMessage()); } + + /** + * Term arguments are validated before the teacher reference is resolved, so a bad term list + * against a hub id fails before anything is downloaded. + */ + @Test + void testRejectsBadTermsBeforeResolvingAHubTeacher(@TempDir Path dir) { + assertEquals("Terms must not be null", + assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill("BAAI/bge-m3", dir.resolve("out"), 256, null, null)) + .getMessage()); + assertEquals("Terms must not contain null", + assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill("BAAI/bge-m3", dir.resolve("out"), 256, + Collections.singletonList(null), null)).getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"&", "!!", " . "}) + void testRejectsATermWithoutALetterOrDigit(String term, @TempDir Path dir) { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelDistiller.distill("BAAI/bge-m3", dir.resolve("out"), 256, List.of(term), + null)); + assertTrue(e.getMessage().contains("no letter or digit"), e.getMessage()); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTermTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTermTest.java new file mode 100644 index 0000000000..3f42831e22 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelTermTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.InvalidFormatException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * A model directory with a term table: term rows pool as single units where they match, the + * subword path is untouched everywhere else, terms appear as similarity-search neighbors, and a + * malformed or mismatched terms file fails loud at load time. + */ +class StaticEmbeddingModelTermTest { + + private static final List VOCABULARY = + List.of("[CLS]", "[SEP]", "[UNK]", "habeas", "corpus", "writ", "law"); + + /** + * The matrix rows: the three special tokens are zero, the content tokens have distinct + * directions, and the two term rows (habeas corpus, replevin) are distinct again. + */ + private static final float[][] ROWS = { + {0f, 0f}, // [CLS] + {0f, 0f}, // [SEP] + {0f, 0f}, // [UNK] + {1f, 0f}, // habeas + {0f, 1f}, // corpus + {2f, 0f}, // writ + {4f, 0f}, // law + {10f, 10f}, // term: habeas corpus + {5f, -5f}, // term: replevin + }; + + /** + * Writes a loadable WordPiece directory, optionally with the two term rows and their + * {@code terms.txt}, and loads it. + * + * @param dir The directory to write into. + * @param withTerms Whether to include the term rows and the terms file. + * @return The loaded model. + * @throws IOException Thrown if writing or loading fails. + */ + private static StaticEmbeddingModel model(Path dir, boolean withTerms) throws IOException { + Files.write(dir.resolve("vocab.txt"), VOCABULARY); + final int rows = withTerms ? ROWS.length : VOCABULARY.size(); + final float[][] matrix = new float[rows][]; + System.arraycopy(ROWS, 0, matrix, 0, rows); + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", matrix)); + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false}"); + Files.writeString(dir.resolve("tokenizer_config.json"), "{\"do_lower_case\":true}"); + if (withTerms) { + Files.write(dir.resolve("terms.txt"), List.of("habeas corpus", "replevin")); + } + return StaticEmbeddingModel.load(dir); + } + + @Test + void testLoadsTheTermTable(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = model(dir, true); + assertEquals(VOCABULARY.size(), model.vocabularySize()); + assertEquals(2, model.termCount()); + } + + @Test + void testAMatchedTermPoolsItsSingleRow(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = model(dir, true); + assertArrayEquals(new float[] {10f, 10f}, model.embed("habeas corpus")); + // Case folding and punctuation between the words do not break the match. + assertArrayEquals(new float[] {10f, 10f}, model.embed("Habeas-Corpus!")); + // A single-word term matches ahead of its (absent) subword pieces. + assertArrayEquals(new float[] {5f, -5f}, model.embed("replevin")); + } + + @Test + void testTermAndPieceRowsPoolTogetherInTextOrder(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = model(dir, true); + // writ -> its piece row; "of" -> [UNK], skipped; "habeas corpus" -> the term row; + // law -> its piece row. Mean of (2,0), (10,10), (4,0). + assertArrayEquals(new float[] {16f / 3, 10f / 3}, model.embed("writ of habeas corpus law")); + } + + @Test + void testTextWithoutAMatchEmbedsExactlyLikeATermlessModel(@TempDir Path dir, + @TempDir Path termless) + throws IOException { + final StaticEmbeddingModel withTerms = model(dir, true); + final StaticEmbeddingModel without = model(termless, false); + // "habeas law" has both words in the vocabulary but matches no term: the two words are not + // adjacent words of any stored phrase. + assertArrayEquals(without.embed("habeas law"), withTerms.embed("habeas law")); + assertArrayEquals(without.embed("the writ, of law."), withTerms.embed("the writ, of law.")); + } + + @Test + void testATermlessModelZeroesWhatOnlyATermRowCouldEmbed(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel without = model(dir, false); + // Without the term table, "replevin" is out of vocabulary entirely. + assertArrayEquals(new float[] {0f, 0f}, without.embed("replevin")); + } + + @Test + void testTermsAreSimilarityNeighbors(@TempDir Path dir) throws IOException { + final StaticEmbeddingModel model = model(dir, true); + final List neighbors = model.mostSimilar("replevin", 1); + assertEquals(1, neighbors.size()); + assertEquals("replevin", neighbors.get(0).token()); + assertEquals("habeas corpus", model.mostSimilar("habeas corpus", 1).get(0).token()); + } + + @Test + void testARowCountMismatchWithTermsFailsLoud(@TempDir Path dir) throws IOException { + Files.write(dir.resolve("vocab.txt"), VOCABULARY); + final float[][] matrix = new float[VOCABULARY.size()][]; + System.arraycopy(ROWS, 0, matrix, 0, VOCABULARY.size()); + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", matrix)); + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false}"); + Files.writeString(dir.resolve("tokenizer_config.json"), "{\"do_lower_case\":true}"); + Files.write(dir.resolve("terms.txt"), List.of("habeas corpus", "replevin")); + + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); + assertTrue(e.getMessage().contains("plus 2 terms"), e.getMessage()); + } + + @Test + void testAMalformedTermsFileFailsLoud(@TempDir Path dir) throws IOException { + Files.write(dir.resolve("vocab.txt"), VOCABULARY); + SafetensorsTestFiles.write(dir.resolve("model.safetensors"), + SafetensorsTestFiles.matrix("embeddings", ROWS)); + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false}"); + Files.writeString(dir.resolve("tokenizer_config.json"), "{\"do_lower_case\":true}"); + // Upper case is not the normalized form the matcher folds to. + Files.write(dir.resolve("terms.txt"), List.of("HABEAS CORPUS", "replevin")); + + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); + assertTrue(e.getMessage().contains("normalized form"), e.getMessage()); + } + + @Test + void testASentencePieceDirectoryLoadsItsTermTable(@TempDir Path dir) throws IOException { + EmbeddingTestFixtures.writeSentencePieceDirectory(dir, List.of("lawbook")); + Files.writeString(dir.resolve("config.json"), + "{\"model_type\":\"model2vec\",\"normalize\":false}"); + + final StaticEmbeddingModel model = StaticEmbeddingModel.load(dir); + assertEquals(1, model.termCount()); + // The fixture's cell formula is row + d * 0.25, and the term owns the row after the + // vocabulary rows. + final float[] expected = new float[EmbeddingTestFixtures.SENTENCEPIECE_DIMENSION]; + for (int d = 0; d < expected.length; d++) { + expected[d] = model.vocabularySize() + d * 0.25f; + } + assertArrayEquals(expected, model.embed("Lawbook")); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java index df1bc9c615..50028a3bb8 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TeacherTokenizerTest.java @@ -492,4 +492,26 @@ void testWriteCleanedRejectsANullFile(@TempDir Path dir) throws IOException { assertEquals("File must not be null", assertThrows( IllegalArgumentException.class, () -> tokenizer.writeCleaned(null)).getMessage()); } + + @Test + void testTermInputSequenceMapsPieceStringsToOriginalIds(@TempDir Path dir) throws IOException { + final TeacherTokenizer tokenizer = TeacherTokenizer.read( + write(dir, "tokenizer.json", WORDPIECE_TEACHER), null); + + // hello and world map to their original ids, an unmapped piece falls to the unknown id, + // and the sequence is wrapped in the post-processor's [CLS]/[SEP] ids. + assertArrayEquals(new long[] {2, 5, 1, 7, 3}, + tokenizer.inputSequence(List.of("hello", "nope", "world"))); + assertEquals("Pieces must not be null", assertThrows(IllegalArgumentException.class, + () -> tokenizer.inputSequence((List) null)).getMessage()); + } + + @Test + void testReadsTheNormalizerLowercaseFlag(@TempDir Path dir) throws IOException { + assertEquals(Boolean.TRUE, TeacherTokenizer.read( + write(dir, "wordpiece.json", WORDPIECE_TEACHER), null).lowerCase()); + // The Unigram teacher states no normalizer, so the flag is unknown. + assertNull(TeacherTokenizer.read( + write(dir, "unigram.json", UNIGRAM_TEACHER), null).lowerCase()); + } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermSegmenterTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermSegmenterTest.java new file mode 100644 index 0000000000..a52ff345e8 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermSegmenterTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The term segmenter's fidelity to the teacher's own tokenization: WordPiece casing and subword + * continuation, unknown-word fallback, delimiter removal, and the SentencePiece path through the + * teacher's trained model file. + */ +class TermSegmenterTest { + + // A WordPiece teacher whose vocabulary can subword-split "corpuses" into corpus + ##es. + private static final String WORDPIECE_TEACHER = + "{\"normalizer\":{\"type\":\"BertNormalizer\",\"lowercase\":true}," + + "\"post_processor\":null," + + "\"model\":{\"type\":\"WordPiece\",\"unk_token\":\"[UNK]\"," + + "\"vocab\":{\"[UNK]\":0,\"[CLS]\":1,\"[SEP]\":2," + + "\"habeas\":3,\"corpus\":4,\"##es\":5}}}"; + + private static TeacherTokenizer wordpieceTeacher(Path dir) throws IOException { + Files.writeString(dir.resolve(ModelFileNames.TOKENIZER_JSON), WORDPIECE_TEACHER); + return TeacherTokenizer.read(dir.resolve(ModelFileNames.TOKENIZER_JSON), null); + } + + @Test + void testSegmentsWithTheTeachersCasingAndSubwords(@TempDir Path dir) throws IOException { + final TermSegmenter segmenter = + TermSegmenter.forTeacher(wordpieceTeacher(dir), dir); + + assertEquals(List.of("habeas", "corpus"), segmenter.pieces("Habeas CORPUS")); + assertEquals(List.of("corpus", "##es"), segmenter.pieces("corpuses")); + } + + @Test + void testDropsTheEncodersDelimitersButKeepsTheUnknownPiece(@TempDir Path dir) + throws IOException { + final TermSegmenter segmenter = + TermSegmenter.forTeacher(wordpieceTeacher(dir), dir); + + // The wrapping [CLS]/[SEP] are the segmenter's own; an out-of-vocabulary word stays as the + // unknown piece, so the teacher still sees a position for it. + assertEquals(List.of("habeas", "[UNK]"), segmenter.pieces("habeas zzz")); + } + + @Test + void testWordpiecePiecesMapBackToTeacherInputIds(@TempDir Path dir) throws IOException { + final TeacherTokenizer teacher = wordpieceTeacher(dir); + final TermSegmenter segmenter = TermSegmenter.forTeacher(teacher, dir); + + // No post-processor, so the sequence is exactly the piece ids in the teacher's id space. + final long[] sequence = teacher.inputSequence(segmenter.pieces("habeas corpuses")); + assertArrayEquals(new long[] {3, 4, 5}, sequence); + } + + @Test + void testAUnigramTeacherSegmentsThroughItsTrainedModelFile(@TempDir Path dir) + throws IOException { + // The trained tiny SentencePiece fixture next to a matching Unigram tokenizer.json. + final byte[] modelBytes; + try (InputStream in = TermSegmenterTest.class + .getResourceAsStream(EmbeddingTestFixtures.TINY_UNIGRAM_RESOURCE)) { + modelBytes = in.readAllBytes(); + } + Files.write(dir.resolve("spiece.model"), modelBytes); + Files.writeString(dir.resolve(ModelFileNames.TOKENIZER_JSON), + "{\"post_processor\":null," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":0," + + "\"vocab\":[[\"\",0.0],[\"▁a\",-1.5],[\"a\",-2.0]]}}"); + final TeacherTokenizer teacher = TeacherTokenizer.read( + dir.resolve(ModelFileNames.TOKENIZER_JSON), null); + + final TermSegmenter segmenter = TermSegmenter.forTeacher(teacher, dir); + final List pieces = segmenter.pieces("a"); + + assertFalse(pieces.isEmpty()); + // Control pieces never appear; every piece is a string the trained model produced. + assertTrue(pieces.stream().noneMatch(p -> p.equals("") || p.equals("")), + pieces.toString()); + } + + @Test + void testAUnigramTeacherWithoutItsModelFileIsRejected(@TempDir Path dir) throws IOException { + Files.writeString(dir.resolve(ModelFileNames.TOKENIZER_JSON), + "{\"post_processor\":null," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":0,\"vocab\":[[\"\",0.0]]}}"); + final TeacherTokenizer teacher = TeacherTokenizer.read( + dir.resolve(ModelFileNames.TOKENIZER_JSON), null); + + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> TermSegmenter.forTeacher(teacher, dir)); + assertTrue(e.getMessage().contains("SentencePiece"), e.getMessage()); + } + + @Test + void testNullArgumentsAreRejected(@TempDir Path dir) throws IOException { + final TeacherTokenizer teacher = wordpieceTeacher(dir); + assertThrows(IllegalArgumentException.class, () -> TermSegmenter.forTeacher(null, dir)); + assertThrows(IllegalArgumentException.class, () -> TermSegmenter.forTeacher(teacher, null)); + final TermSegmenter segmenter = TermSegmenter.forTeacher(teacher, dir); + assertThrows(IllegalArgumentException.class, () -> segmenter.pieces(null)); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermTableTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermTableTest.java new file mode 100644 index 0000000000..aee76a3c90 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/TermTableTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import opennlp.tools.util.InvalidFormatException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The term table's normalization contract, its validation of stored terms, and the greedy + * longest-first matching over word runs. + */ +class TermTableTest { + + private static final String SOURCE = "terms.txt"; + + private static TermTable table(String... terms) throws InvalidFormatException { + return TermTable.of(List.of(terms), 10, SOURCE); + } + + @ParameterizedTest + @CsvSource(delimiter = ';', value = { + "habeas corpus;habeas corpus", + "Habeas Corpus;habeas corpus", + "habeas-corpus!;habeas corpus", + "' writ OF Habeas ';writ of habeas", + "res judicata.;res judicata", + "42 USC 1983;42 usc 1983" + }) + void testNormalizeTermFoldsAndJoinsWordRuns(String raw, String expected) { + assertEquals(expected, TermTable.normalizeTerm(raw)); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "&!.", "--"}) + void testNormalizeTermOfTextWithoutWordsIsEmpty(String raw) { + assertEquals("", TermTable.normalizeTerm(raw)); + } + + @Test + void testNormalizeTermFoldsSupplementaryPlaneLetters() { + // DESERET CAPITAL LETTER LONG I (U+10400) is a cased letter outside the BMP; its lower-case + // form is U+10428, one code point, so the word run survives the fold intact. + final String capital = new String(Character.toChars(0x10400)); + final String small = new String(Character.toChars(0x10428)); + assertEquals(small + "x", TermTable.normalizeTerm(capital + "x")); + } + + @Test + void testRejectsATermThatIsNotNormalized() { + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> table("HABEAS CORPUS")); + assertTrue(e.getMessage().contains("HABEAS CORPUS"), e.getMessage()); + assertTrue(e.getMessage().contains(SOURCE), e.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"", "habeas corpus", " habeas", "habeas "}) + void testRejectsMalformedTermForms(String term) { + assertThrows(InvalidFormatException.class, () -> table(term)); + } + + @Test + void testRejectsADuplicateTerm() { + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> table("habeas corpus", "habeas corpus")); + assertTrue(e.getMessage().contains("more than once"), e.getMessage()); + } + + @Test + void testRejectsNullArguments() { + assertThrows(IllegalArgumentException.class, () -> TermTable.of(null, 0, SOURCE)); + assertThrows(IllegalArgumentException.class, () -> TermTable.normalizeTerm(null)); + } + + @Test + void testTermsOwnRowsFromTheFirstRowOnward() throws InvalidFormatException { + final TermTable table = table("habeas corpus", "replevin"); + assertEquals(2, table.size()); + assertEquals("habeas corpus", table.term(10)); + assertEquals("replevin", table.term(11)); + assertThrows(IllegalArgumentException.class, () -> table.term(9)); + assertThrows(IllegalArgumentException.class, () -> table.term(12)); + } + + @Test + void testMatchesFoldCaseAndSpanPunctuation() throws InvalidFormatException { + final TermTable table = table("habeas corpus"); + final List matches = table.matches("The writ of Habeas-Corpus, granted."); + assertEquals(1, matches.size()); + assertEquals(10, matches.get(0).row()); + assertEquals("Habeas-Corpus", "The writ of Habeas-Corpus, granted." + .substring(matches.get(0).start(), matches.get(0).end())); + } + + @Test + void testTheLongestTermWinsAndConsumesItsWords() throws InvalidFormatException { + final TermTable table = table("habeas corpus", "writ of habeas corpus", "corpus"); + final List matches = table.matches("a writ of habeas corpus indeed"); + // The four-word term wins over both shorter terms, and its words are consumed: the inner + // "habeas corpus" and "corpus" do not match again. + assertEquals(1, matches.size()); + assertEquals(11, matches.get(0).row()); + } + + @Test + void testMatchingContinuesAfterAConsumedTerm() throws InvalidFormatException { + final TermTable table = table("habeas corpus", "replevin"); + final List matches = table.matches("habeas corpus then replevin"); + assertEquals(2, matches.size()); + assertEquals(10, matches.get(0).row()); + assertEquals(11, matches.get(1).row()); + assertTrue(matches.get(0).end() <= matches.get(1).start()); + } + + @Test + void testWordsSeparatedByOtherWordsDoNotMatchAPhrase() throws InvalidFormatException { + final TermTable table = table("habeas corpus"); + assertTrue(table.matches("habeas late corpus").isEmpty()); + } + + @Test + void testAnEmptyTableMatchesNothing() throws InvalidFormatException { + assertTrue(table().matches("habeas corpus").isEmpty()); + } + + @Test + void testMatchesRejectsNullText() throws InvalidFormatException { + final TermTable table = table("habeas corpus"); + assertThrows(IllegalArgumentException.class, () -> table.matches(null)); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java index f69c2b4bf3..dad82c8b85 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java @@ -70,8 +70,9 @@ void testDistillHelpNamesEveryParameter() { assertTrue(help.contains("-teacher hf-id-or-path"), help); assertTrue(help.contains("-out dir"), help); - // The optional parameter is bracketed, so a user can see it may be omitted. + // The optional parameters are bracketed, so a user can see they may be omitted. assertTrue(help.contains("[-pcaDims "), help); + assertTrue(help.contains("[-terms "), help); } @Test From 18d4d007abe56c12b693da22e4de7efa17595023 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 23 Jul 2026 23:40:30 -0400 Subject: [PATCH 71/82] OPENNLP-1895: Quantized embedding matrix core: seeded rotation, Lloyd-Max grids, packed codes --- .../opennlp/embeddings/GaussianQuantizer.java | 230 +++++++ .../opennlp/embeddings/HadamardRotation.java | 188 ++++++ .../embeddings/QuantizedEmbeddingMatrix.java | 565 ++++++++++++++++++ .../embeddings/GaussianQuantizerTest.java | 123 ++++ .../embeddings/HadamardRotationTest.java | 155 +++++ .../QuantizedEmbeddingMatrixTest.java | 289 +++++++++ 6 files changed, 1550 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HadamardRotation.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/GaussianQuantizerTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java new file mode 100644 index 0000000000..898ac77433 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.Arrays; + +/** + * An optimal scalar quantizer for standard-normal values: {@code 2^bits} representation levels + * minimizing the mean squared error over {@code N(0,1)}, with encoding by nearest level. The + * coordinates of a {@link HadamardRotation rotated} unit vector, scaled by the square root of the + * padded dimension, follow this distribution closely, which is what makes one fixed grid + * near-optimal for every coordinate of every vector (Zandieh et al., TurboQuant: Online Vector + * Quantization with Near-optimal Distortion Rate, arXiv:2504.19874). + * + *

The levels are the classic Lloyd-Max quantizer of the Gaussian (Max, Quantizing for + * minimum distortion, IRE Transactions on Information Theory, 1960), computed here by Lloyd + * iteration over a fine discretization of the density rather than copied from published tables, + * so the derivation is in this file and reproducible. Computed grids are cached per bit width. + * Encoding compares against the midpoints between adjacent levels, which is exactly the + * nearest-level rule for a sorted grid.

+ * + *

A quantized file stores its grid, and reading rebuilds the quantizer from the stored levels + * through {@link #fromLevels(float[])}, so decoding never depends on this derivation matching the + * one that encoded the file.

+ * + *

Instances are immutable and safe for concurrent use.

+ */ +final class GaussianQuantizer { + + /** The smallest supported bit width. */ + static final int MIN_BITS = 2; + + /** The largest supported bit width. */ + static final int MAX_BITS = 4; + + // The density is discretized on [-RANGE, RANGE]; beyond eight standard deviations the + // remaining mass (~1e-15) is far below the iteration tolerance. + private static final double LLOYD_RANGE = 8.0; + private static final int LLOYD_SAMPLES = 200_001; + private static final double LLOYD_TOLERANCE = 1e-10; + private static final int LLOYD_MAX_ITERATIONS = 1_000; + + private static final GaussianQuantizer[] CACHE = new GaussianQuantizer[MAX_BITS + 1]; + + private final float[] levels; + // Midpoints between adjacent levels: level i is nearest exactly when the value lies in + // (thresholds[i-1], thresholds[i]], with the outermost intervals unbounded. + private final float[] thresholds; + + /** + * Holds a validated grid; callers reach this through {@link #forBits(int)} or + * {@link #fromLevels(float[])}. + * + * @param levels The representation levels, ascending. + */ + private GaussianQuantizer(float[] levels) { + this.levels = levels; + this.thresholds = new float[levels.length - 1]; + for (int i = 0; i < thresholds.length; i++) { + thresholds[i] = (levels[i] + levels[i + 1]) / 2f; + } + } + + /** + * {@return the quantizer for a bit width, computed once and cached} + * + * @param bits The bit width. Must be between {@link #MIN_BITS} and {@link #MAX_BITS}. + * @throws IllegalArgumentException Thrown if {@code bits} is outside the supported range. + */ + static GaussianQuantizer forBits(int bits) { + requireSupportedBits(bits); + synchronized (CACHE) { + if (CACHE[bits] == null) { + CACHE[bits] = new GaussianQuantizer(lloydMaxLevels(1 << bits)); + } + return CACHE[bits]; + } + } + + /** + * {@return a quantizer over a stored grid, as read back from a quantized file} + * + * @param levels The representation levels, strictly ascending and finite, of a power-of-two + * length between {@code 2^MIN_BITS} and {@code 2^MAX_BITS}. The array is copied. + * @throws IllegalArgumentException Thrown if {@code levels} is {@code null}, of an unsupported + * length, not strictly ascending, or not finite. + */ + static GaussianQuantizer fromLevels(float[] levels) { + if (levels == null) { + throw new IllegalArgumentException("Levels must not be null"); + } + if (levels.length != Integer.highestOneBit(levels.length) + || levels.length < 1 << MIN_BITS || levels.length > 1 << MAX_BITS) { + throw new IllegalArgumentException("Levels must have a power-of-two length between " + + (1 << MIN_BITS) + " and " + (1 << MAX_BITS) + ", got " + levels.length); + } + for (int i = 0; i < levels.length; i++) { + if (!Float.isFinite(levels[i])) { + throw new IllegalArgumentException("Level " + i + " is not finite: " + levels[i]); + } + if (i > 0 && levels[i] <= levels[i - 1]) { + throw new IllegalArgumentException("Levels must be strictly ascending, but level " + + i + " (" + levels[i] + ") is not above level " + (i - 1) + + " (" + levels[i - 1] + ")"); + } + } + return new GaussianQuantizer(Arrays.copyOf(levels, levels.length)); + } + + /** + * Requires a bit width within the supported range. + * + * @param bits The bit width to check. + * @throws IllegalArgumentException Thrown if {@code bits} is outside the supported range. + */ + static void requireSupportedBits(int bits) { + if (bits < MIN_BITS || bits > MAX_BITS) { + throw new IllegalArgumentException("Bits must be between " + MIN_BITS + " and " + + MAX_BITS + ", got " + bits); + } + } + + /** {@return the number of representation levels} */ + int levelCount() { + return levels.length; + } + + /** + * {@return the representation level of a code} + * + * @param code The code, between 0 and {@code levelCount() - 1}. + */ + float level(int code) { + return levels[code]; + } + + /** {@return a copy of the representation levels, ascending} */ + float[] levels() { + return Arrays.copyOf(levels, levels.length); + } + + /** + * {@return the code of the representation level nearest a value} Ties at a midpoint take the + * lower level, a fixed convention so encoding is deterministic. + * + * @param value The value to encode. + */ + int encode(float value) { + int low = 0; + int high = thresholds.length; + while (low < high) { + final int middle = (low + high) >>> 1; + if (value <= thresholds[middle]) { + high = middle; + } else { + low = middle + 1; + } + } + return low; + } + + /** + * {@return the Lloyd-Max representation levels for the standard normal} Lloyd iteration over a + * fine discretization of the density: assign each sample to its nearest level, move each level + * to the probability-weighted mean of its samples, repeat to a fixed point. The discretization, + * tolerance, and iteration cap are constants of this file, so the result is deterministic. + * + * @param levelCount The number of levels, a power of two. + */ + private static float[] lloydMaxLevels(int levelCount) { + final double step = 2 * LLOYD_RANGE / (LLOYD_SAMPLES - 1); + final double[] samples = new double[LLOYD_SAMPLES]; + final double[] weights = new double[LLOYD_SAMPLES]; + for (int i = 0; i < LLOYD_SAMPLES; i++) { + samples[i] = -LLOYD_RANGE + i * step; + weights[i] = Math.exp(-samples[i] * samples[i] / 2); + } + // Initial levels: evenly spaced over the central mass; Lloyd converges to the optimum for + // the log-concave Gaussian regardless of the starting spread. + final double[] levels = new double[levelCount]; + for (int i = 0; i < levelCount; i++) { + levels[i] = -3.0 + 6.0 * (i + 0.5) / levelCount; + } + final double[] weightSums = new double[levelCount]; + final double[] weightedValueSums = new double[levelCount]; + for (int iteration = 0; iteration < LLOYD_MAX_ITERATIONS; iteration++) { + Arrays.fill(weightSums, 0); + Arrays.fill(weightedValueSums, 0); + int level = 0; + for (int i = 0; i < LLOYD_SAMPLES; i++) { + while (level < levelCount - 1 + && samples[i] > (levels[level] + levels[level + 1]) / 2) { + level++; + } + weightSums[level] += weights[i]; + weightedValueSums[level] += weights[i] * samples[i]; + } + double largestMove = 0; + for (int i = 0; i < levelCount; i++) { + if (weightSums[i] > 0) { + final double moved = weightedValueSums[i] / weightSums[i]; + largestMove = Math.max(largestMove, Math.abs(moved - levels[i])); + levels[i] = moved; + } + } + if (largestMove < LLOYD_TOLERANCE) { + break; + } + } + final float[] result = new float[levelCount]; + for (int i = 0; i < levelCount; i++) { + result[i] = (float) levels[i]; + } + return result; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HadamardRotation.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HadamardRotation.java new file mode 100644 index 0000000000..9a4abd89a4 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/HadamardRotation.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +/** + * A seeded randomized Hadamard rotation: a deterministic random sign flip per coordinate followed + * by the normalized fast Walsh-Hadamard transform. Rotating a vector this way spreads its energy + * evenly across coordinates, so each coordinate of a rotated unit vector is approximately + * Gaussian with variance {@code 1/paddedDimension} and nearly independent of the others, which is + * the property {@link GaussianQuantizer}'s per-coordinate grids rely on. + * + *

The transform is orthonormal, so it preserves norms and inner products exactly (up to float + * rounding): two vectors rotated with the same instance have the same dot product as the + * originals, which lets similarity math stay in rotated space and never pay for the inverse. + * Writing {@code S} for the sign flip and {@code H} for the normalized Walsh-Hadamard matrix + * (which is its own inverse), the rotation is {@code H·S} and its inverse is {@code S·H}: the + * same two operations applied in the opposite order, so no second table is needed.

+ * + *

The Walsh-Hadamard transform needs a power-of-two length, so vectors are padded with zeros + * from their original dimension up to {@link #paddedDimension(int)}. The sign flips derive from + * the seed through an in-file + * splitmix64 step, not through a JDK + * generator, so the same seed produces the same rotation on every JVM and release; the seed is + * stored in the quantized file and the rotation is rebuilt from it on load.

+ * + *

Instances are immutable and safe for concurrent use.

+ */ +final class HadamardRotation { + + private static final long SPLITMIX64_GOLDEN_GAMMA = 0x9E3779B97F4A7C15L; + + private final int paddedDimension; + // True where the coordinate is negated before (rotate) or after (inverse) the transform. + private final boolean[] flip; + private final float inverseSquareRoot; + + /** + * Creates the rotation for vectors of the given original dimension. + * + * @param dimension The original vector dimension. Must be at least 1. + * @param seed The seed the sign flips derive from. + * @throws IllegalArgumentException Thrown if {@code dimension} is less than 1. + */ + HadamardRotation(int dimension, long seed) { + if (dimension < 1) { + throw new IllegalArgumentException("Dimension must be at least 1, got " + dimension); + } + this.paddedDimension = paddedDimension(dimension); + this.flip = new boolean[paddedDimension]; + long state = seed; + long bits = 0; + for (int i = 0; i < paddedDimension; i++) { + if ((i & 63) == 0) { + state += SPLITMIX64_GOLDEN_GAMMA; + bits = splitmix64(state); + } + flip[i] = (bits & 1L) != 0; + bits >>>= 1; + } + this.inverseSquareRoot = (float) (1.0 / Math.sqrt(paddedDimension)); + } + + /** + * {@return the power-of-two length vectors are padded to before the transform} + * + * @param dimension The original vector dimension. Must be at least 1. + * @throws IllegalArgumentException Thrown if {@code dimension} is less than 1. + */ + static int paddedDimension(int dimension) { + if (dimension < 1) { + throw new IllegalArgumentException("Dimension must be at least 1, got " + dimension); + } + if (dimension > 1 << 30) { + throw new IllegalArgumentException("Dimension must be at most " + (1 << 30) + + " so the padded length stays an int power of two, got " + dimension); + } + final int highestOneBit = Integer.highestOneBit(dimension); + return highestOneBit == dimension ? dimension : highestOneBit << 1; + } + + /** {@return the power-of-two length this instance transforms} */ + int paddedDimension() { + return paddedDimension; + } + + /** + * Rotates a vector in place: sign flips, then the normalized Walsh-Hadamard transform. + * + * @param vector The vector to rotate. Must not be {@code null} and must have length + * {@link #paddedDimension()}. + * @throws IllegalArgumentException Thrown if {@code vector} is {@code null} or has the wrong + * length. + */ + void rotate(float[] vector) { + requirePaddedLength(vector); + for (int i = 0; i < paddedDimension; i++) { + if (flip[i]) { + vector[i] = -vector[i]; + } + } + walshHadamard(vector); + } + + /** + * Applies the inverse rotation in place: the normalized Walsh-Hadamard transform, then the + * sign flips. + * + * @param vector The rotated vector. Must not be {@code null} and must have length + * {@link #paddedDimension()}. + * @throws IllegalArgumentException Thrown if {@code vector} is {@code null} or has the wrong + * length. + */ + void inverse(float[] vector) { + requirePaddedLength(vector); + walshHadamard(vector); + for (int i = 0; i < paddedDimension; i++) { + if (flip[i]) { + vector[i] = -vector[i]; + } + } + } + + /** + * Requires the vector to be non-null and of the padded length. + * + * @param vector The vector to check. + * @throws IllegalArgumentException Thrown if {@code vector} is {@code null} or has the wrong + * length. + */ + private void requirePaddedLength(float[] vector) { + if (vector == null) { + throw new IllegalArgumentException("Vector must not be null"); + } + if (vector.length != paddedDimension) { + throw new IllegalArgumentException("Vector has length " + vector.length + + " but this rotation transforms length " + paddedDimension); + } + } + + /** + * The in-place normalized fast Walsh-Hadamard transform, {@code O(n log n)} butterflies + * followed by a {@code 1/sqrt(n)} scale so the transform is orthonormal and self-inverse. + * + * @param vector The vector to transform, of the padded length. + */ + private void walshHadamard(float[] vector) { + for (int half = 1; half < paddedDimension; half <<= 1) { + for (int block = 0; block < paddedDimension; block += half << 1) { + for (int i = block; i < block + half; i++) { + final float a = vector[i]; + final float b = vector[i + half]; + vector[i] = a + b; + vector[i + half] = a - b; + } + } + } + for (int i = 0; i < paddedDimension; i++) { + vector[i] *= inverseSquareRoot; + } + } + + /** + * {@return the splitmix64 mix of a state word} The finalizer of the splitmix64 generator, + * reproduced here so the bit stream is fixed by this file rather than by a JDK class. + * + * @param state The state word to mix. + */ + private static long splitmix64(long state) { + long z = state; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java new file mode 100644 index 0000000000..9e9a6595ee --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java @@ -0,0 +1,565 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import opennlp.tools.commons.ThreadSafe; + +/** + * An embedding matrix quantized to {@code 2}-{@code 4} bits per dimension, following the + * TurboQuant construction (Zandieh, Daliri, Hadian, Mirrokni, TurboQuant: Online Vector + * Quantization with Near-optimal Distortion Rate, arXiv:2504.19874): each row is rotated by + * a seeded {@link HadamardRotation}, so its coordinates become near-independent and + * near-Gaussian, and each rotated coordinate is encoded independently with the + * {@link GaussianQuantizer} grid of the chosen bit width. A row decodes to a per-row scale times + * its grid levels; the scale is least-squares fitted per row, which strictly reduces the squared + * error of the fixed grid. + * + *

The storage is {@code bits} per dimension plus one float per row, against 32 bits per + * dimension for the float matrix: a 500,000-row, 300-dimension table shrinks from roughly 600 MB + * to 77 MB at 4 bits (the padded dimension, 512 here, is what is stored). The workload this + * serves is memory-bound row gathering, so reading fewer bytes is also the throughput lever.

+ * + *

Rows live in rotated space, and the cheap operations stay there: the rotation is + * orthonormal, so dot products and norms of rotated vectors equal those of the originals, and + * pooling commutes with it because rotation is linear. A consumer embeds text by summing rows + * with {@link #addRowRotated(int, float, float[])} and applying {@link #toOriginal(float[])} + * once per text, not once per row; a similarity scan rotates the query once with + * {@link #rotate(float[])} and scores every row with {@link #dotRotated(int, float[])}, never + * leaving rotated space. {@link #decodeRow(int)} exists for callers that need one original-space + * row and for measuring reconstruction quality.

+ * + *

The file format is self-describing: it stores the grid levels and the rotation seed, so a + * reader reconstructs exactly the decoder the writer used and never depends on this class's grid + * derivation staying fixed. Quantizing is deterministic: the same matrix, bit width, and seed + * produce the same file bytes on every JVM.

+ * + *

Instances are immutable and safe for concurrent use after construction.

+ */ +@ThreadSafe +public final class QuantizedEmbeddingMatrix { + + /** The smallest supported bit width. */ + public static final int MIN_BITS = GaussianQuantizer.MIN_BITS; + + /** The largest supported bit width. */ + public static final int MAX_BITS = GaussianQuantizer.MAX_BITS; + + // "ONQ1": OpenNLP quantized matrix, format 1. + private static final int MAGIC = 0x4F4E5131; + + private final int rowCount; + private final int dimension; + private final int paddedDimension; + private final int bits; + private final long seed; + private final int rowBytes; + private final GaussianQuantizer quantizer; + private final HadamardRotation rotation; + // One scale per row: decoded rotated coordinate i of a row is scale * level(code_i). + private final float[] scales; + // Packed codes, row-major: row r's code i occupies bits [i*bits, (i+1)*bits) of the row's + // rowBytes region, little-endian within the region. + private final byte[] codes; + // The L2 norm of each decoded original-space row. Quantization noise leaves some energy in + // the padding coordinates, which truncation drops, so this is computed exactly at quantize + // time (one inverse rotation per row) and stored in the file rather than recomputed from the + // codes on load. + private final float[] decodedNorms; + + /** + * Holds validated state; callers reach this through {@link #quantize} or {@link #read}. + */ + private QuantizedEmbeddingMatrix(int rowCount, int dimension, int bits, long seed, + GaussianQuantizer quantizer, float[] scales, byte[] codes, + float[] decodedNorms) { + this.rowCount = rowCount; + this.dimension = dimension; + this.paddedDimension = HadamardRotation.paddedDimension(dimension); + this.bits = bits; + this.seed = seed; + this.rowBytes = (paddedDimension * bits + 7) / 8; + this.quantizer = quantizer; + this.rotation = new HadamardRotation(dimension, seed); + this.scales = scales; + this.codes = codes; + this.decodedNorms = decodedNorms; + } + + /** + * Quantizes a float matrix. + * + * @param rowMajor The matrix, row-major, {@code rowCount * dimension} floats. Must not be + * {@code null} and every value must be finite. + * @param rowCount The number of rows. Must be at least 1. + * @param dimension The row width. Must be at least 1. + * @param bits The bit width per (padded) dimension. Must be between {@link #MIN_BITS} and + * {@link #MAX_BITS}. + * @param seed The rotation seed. Any value; stored in the file so decoding rebuilds the + * same rotation. + * @return The quantized matrix. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or out of range, the + * array length does not match {@code rowCount * dimension}, or a value is not finite. + */ + public static QuantizedEmbeddingMatrix quantize(float[] rowMajor, int rowCount, int dimension, + int bits, long seed) { + if (rowMajor == null) { + throw new IllegalArgumentException("RowMajor must not be null"); + } + if (rowCount < 1) { + throw new IllegalArgumentException("RowCount must be at least 1, got " + rowCount); + } + if (dimension < 1) { + throw new IllegalArgumentException("Dimension must be at least 1, got " + dimension); + } + if (rowMajor.length != (long) rowCount * dimension) { + throw new IllegalArgumentException("RowMajor has " + rowMajor.length + " floats but " + + rowCount + " rows of dimension " + dimension + " need " + + ((long) rowCount * dimension)); + } + GaussianQuantizer.requireSupportedBits(bits); + final GaussianQuantizer quantizer = GaussianQuantizer.forBits(bits); + final HadamardRotation rotation = new HadamardRotation(dimension, seed); + final int paddedDimension = rotation.paddedDimension(); + final int rowBytes = (paddedDimension * bits + 7) / 8; + requireStorableSize(rowCount, rowBytes); + final float[] scales = new float[rowCount]; + final byte[] codes = new byte[rowCount * rowBytes]; + final float[] decodedNorms = new float[rowCount]; + final float[] rotated = new float[paddedDimension]; + final float[] decoded = new float[paddedDimension]; + final double squareRootOfPadded = Math.sqrt(paddedDimension); + for (int row = 0; row < rowCount; row++) { + final int base = row * dimension; + double sumOfSquares = 0; + for (int d = 0; d < dimension; d++) { + final float value = rowMajor[base + d]; + if (!Float.isFinite(value)) { + throw new IllegalArgumentException("Row " + row + " has a non-finite value at " + + "dimension " + d + ": " + value + "; a quantized matrix cannot represent it"); + } + rotated[d] = value; + sumOfSquares += (double) value * value; + } + Arrays.fill(rotated, dimension, paddedDimension, 0f); + final double norm = Math.sqrt(sumOfSquares); + if (norm == 0) { + // A zero row has no direction; a zero scale decodes it to zero whatever the codes say, + // and encoding zeros keeps the bytes deterministic. + scales[row] = 0f; + final int zeroCode = quantizer.encode(0f); + for (int i = 0; i < paddedDimension; i++) { + writeCode(codes, row * rowBytes, bits, i, zeroCode); + } + continue; + } + rotation.rotate(rotated); + // Standardized coordinates are near N(0,1); encode each against the grid, then fit the + // one free scale to the row: the alpha minimizing ||z - alpha*g||^2 is (z.g)/(g.g). + final double standardize = squareRootOfPadded / norm; + double gridDot = 0; + double gridSquares = 0; + for (int i = 0; i < paddedDimension; i++) { + final float standardized = (float) (rotated[i] * standardize); + final int code = quantizer.encode(standardized); + writeCode(codes, row * rowBytes, bits, i, code); + final float level = quantizer.level(code); + gridDot += (double) standardized * level; + gridSquares += (double) level * level; + } + final double fitted = gridDot > 0 ? gridDot / gridSquares : 1.0; + scales[row] = (float) (norm / squareRootOfPadded * fitted); + // The decoded original-space norm: quantization noise leaves energy in the padding + // coordinates and truncation drops it, so the norm is measured on the truncated decode, + // not on the rotated codes. + for (int i = 0; i < paddedDimension; i++) { + decoded[i] = scales[row] * quantizer.level(readCode(codes, row * rowBytes, bits, i)); + } + rotation.inverse(decoded); + double decodedSumOfSquares = 0; + for (int d = 0; d < dimension; d++) { + decodedSumOfSquares += (double) decoded[d] * decoded[d]; + } + decodedNorms[row] = (float) Math.sqrt(decodedSumOfSquares); + } + return new QuantizedEmbeddingMatrix(rowCount, dimension, bits, seed, quantizer, scales, + codes, decodedNorms); + } + + /** + * Requires the packed code array to fit in one Java array. + * + * @param rowCount The number of rows. + * @param rowBytes The packed bytes per row. + * @throws IllegalArgumentException Thrown if the total exceeds what an array can hold. + */ + private static void requireStorableSize(int rowCount, int rowBytes) { + if ((long) rowCount * rowBytes > Integer.MAX_VALUE - 8) { + throw new IllegalArgumentException("The packed codes need " + ((long) rowCount * rowBytes) + + " bytes, more than one array can hold; split the matrix"); + } + } + + /** {@return the number of rows} */ + public int rowCount() { + return rowCount; + } + + /** {@return the original row width} */ + public int dimension() { + return dimension; + } + + /** {@return the power-of-two width rows are padded to in rotated space} */ + public int paddedDimension() { + return paddedDimension; + } + + /** {@return the bit width per padded dimension} */ + public int bits() { + return bits; + } + + /** {@return the rotation seed} */ + public long seed() { + return seed; + } + + /** + * Rotates an original-space vector into this matrix's rotated space, padding it first. Rotate + * a query once, then score rows against it with {@link #dotRotated(int, float[])}. + * + * @param vector The original-space vector. Must not be {@code null} and must have length + * {@link #dimension()}. + * @return A new array of length {@link #paddedDimension()} holding the rotated vector. + * @throws IllegalArgumentException Thrown if {@code vector} is {@code null} or has the wrong + * length. + */ + public float[] rotate(float[] vector) { + if (vector == null) { + throw new IllegalArgumentException("Vector must not be null"); + } + if (vector.length != dimension) { + throw new IllegalArgumentException("Vector has length " + vector.length + + " but this matrix has dimension " + dimension); + } + final float[] padded = new float[paddedDimension]; + System.arraycopy(vector, 0, padded, 0, dimension); + rotation.rotate(padded); + return padded; + } + + /** + * Maps a rotated-space vector back to original space. Apply this once per pooled result, after + * accumulating rows with {@link #addRowRotated(int, float, float[])}; rotation is linear, so + * the sum of rotated rows is the rotation of the summed rows. + * + * @param rotated The rotated-space vector. Must not be {@code null} and must have length + * {@link #paddedDimension()}. Not modified. + * @return A new array of length {@link #dimension()} holding the original-space vector. + * @throws IllegalArgumentException Thrown if {@code rotated} is {@code null} or has the wrong + * length. + */ + public float[] toOriginal(float[] rotated) { + if (rotated == null) { + throw new IllegalArgumentException("Rotated must not be null"); + } + if (rotated.length != paddedDimension) { + throw new IllegalArgumentException("Rotated has length " + rotated.length + + " but this matrix's padded dimension is " + paddedDimension); + } + final float[] copy = rotated.clone(); + rotation.inverse(copy); + final float[] original = new float[dimension]; + System.arraycopy(copy, 0, original, 0, dimension); + return original; + } + + /** + * Adds a decoded row, times a weight, onto a rotated-space accumulator. This is the pooling + * primitive: decode stays in rotated space and costs one grid lookup per coordinate. + * + * @param row The row to add. Must be between 0 and {@code rowCount() - 1}. + * @param weight The weight to multiply the row by. + * @param sum The accumulator. Must not be {@code null} and must have length + * {@link #paddedDimension()}. + * @throws IllegalArgumentException Thrown if {@code row} is out of range or {@code sum} is + * {@code null} or has the wrong length. + */ + public void addRowRotated(int row, float weight, float[] sum) { + requireRow(row); + if (sum == null) { + throw new IllegalArgumentException("Sum must not be null"); + } + if (sum.length != paddedDimension) { + throw new IllegalArgumentException("Sum has length " + sum.length + + " but this matrix's padded dimension is " + paddedDimension); + } + final float scaledWeight = scales[row] * weight; + final int base = row * rowBytes; + for (int i = 0; i < paddedDimension; i++) { + sum[i] += scaledWeight * quantizer.level(readCode(codes, base, bits, i)); + } + } + + /** + * The dot product of a decoded row with a rotated-space query. Because the rotation is + * orthonormal, this equals the original-space dot product of the decoded row with the + * un-rotated query, up to float rounding. + * + * @param row The row to score. Must be between 0 and {@code rowCount() - 1}. + * @param rotatedQuery The query in rotated space, as returned by {@link #rotate(float[])}. + * Must not be {@code null} and must have length + * {@link #paddedDimension()}. + * @return The dot product. + * @throws IllegalArgumentException Thrown if {@code row} is out of range or + * {@code rotatedQuery} is {@code null} or has the wrong length. + */ + public double dotRotated(int row, float[] rotatedQuery) { + requireRow(row); + if (rotatedQuery == null) { + throw new IllegalArgumentException("RotatedQuery must not be null"); + } + if (rotatedQuery.length != paddedDimension) { + throw new IllegalArgumentException("RotatedQuery has length " + rotatedQuery.length + + " but this matrix's padded dimension is " + paddedDimension); + } + final int base = row * rowBytes; + double dot = 0; + for (int i = 0; i < paddedDimension; i++) { + dot += (double) rotatedQuery[i] * quantizer.level(readCode(codes, base, bits, i)); + } + return dot * scales[row]; + } + + /** + * {@return the L2 norm of the decoded original-space row, for cosine scoring} Computed + * exactly at quantize time and stored in the file: quantization noise leaves some energy in + * the padding coordinates, which decoding truncates away, so this norm matches + * {@link #decodeRow(int)}'s result rather than the rotated codes. + * + * @param row The row. Must be between 0 and {@code rowCount() - 1}. + * @throws IllegalArgumentException Thrown if {@code row} is out of range. + */ + public double rowNorm(int row) { + requireRow(row); + return decodedNorms[row]; + } + + /** + * Decodes one row back to original space. This pays the inverse rotation for a single row; + * pooling and scanning callers should stay in rotated space instead (see the class comment). + * + * @param row The row to decode. Must be between 0 and {@code rowCount() - 1}. + * @return A new array of length {@link #dimension()} holding the decoded row. + * @throws IllegalArgumentException Thrown if {@code row} is out of range. + */ + public float[] decodeRow(int row) { + requireRow(row); + final float[] rotated = new float[paddedDimension]; + addRowRotated(row, 1f, rotated); + return toOriginal(rotated); + } + + /** + * Requires a row index in range. + * + * @param row The row index to check. + * @throws IllegalArgumentException Thrown if {@code row} is out of range. + */ + private void requireRow(int row) { + if (row < 0 || row >= rowCount) { + throw new IllegalArgumentException("Row must be between 0 and " + (rowCount - 1) + + ", got " + row); + } + } + + /** + * Writes this matrix to a file, deterministically: the same matrix, bit width, and seed + * produce the same bytes. + * + * @param file The file to write. Must not be {@code null}; an existing file is replaced. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null}. + * @throws IOException Thrown if writing fails. + */ + public void write(Path file) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + try (OutputStream out = Files.newOutputStream(file); + DataOutputStream data = new DataOutputStream(new BufferedOutputStream(out))) { + data.writeInt(MAGIC); + data.writeInt(rowCount); + data.writeInt(dimension); + data.writeInt(bits); + data.writeLong(seed); + final float[] levels = quantizer.levels(); + data.writeInt(levels.length); + for (final float level : levels) { + data.writeFloat(level); + } + for (final float scale : scales) { + data.writeFloat(scale); + } + for (final float decodedNorm : decodedNorms) { + data.writeFloat(decodedNorm); + } + data.write(codes); + } + } + + /** + * Reads a matrix written by {@link #write(Path)}. The stored grid and seed rebuild exactly the + * decoder the writer used. + * + * @param file The file to read. Must not be {@code null}. + * @return The quantized matrix. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or its content is + * not a quantized matrix of a supported version. + * @throws IOException Thrown if reading fails or the file is truncated. + */ + public static QuantizedEmbeddingMatrix read(Path file) throws IOException { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + try (InputStream in = Files.newInputStream(file); + DataInputStream data = new DataInputStream(new BufferedInputStream(in))) { + final int magic = data.readInt(); + if (magic != MAGIC) { + throw new IllegalArgumentException(file + " is not a quantized embedding matrix " + + "(magic 0x" + Integer.toHexString(magic) + ", expected 0x" + + Integer.toHexString(MAGIC) + ")"); + } + final int rowCount = data.readInt(); + if (rowCount < 1) { + throw new IllegalArgumentException(file + " declares " + rowCount + " rows; a " + + "quantized matrix has at least 1"); + } + final int dimension = data.readInt(); + if (dimension < 1) { + throw new IllegalArgumentException(file + " declares dimension " + dimension + "; a " + + "quantized matrix's dimension is at least 1"); + } + final int bits = data.readInt(); + GaussianQuantizer.requireSupportedBits(bits); + final long seed = data.readLong(); + final int levelCount = data.readInt(); + if (levelCount != 1 << bits) { + throw new IllegalArgumentException(file + " declares " + levelCount + " grid levels " + + "for " + bits + " bits; expected " + (1 << bits)); + } + final float[] levels = new float[levelCount]; + for (int i = 0; i < levelCount; i++) { + levels[i] = data.readFloat(); + } + final GaussianQuantizer quantizer = GaussianQuantizer.fromLevels(levels); + final float[] scales = new float[rowCount]; + for (int row = 0; row < rowCount; row++) { + scales[row] = data.readFloat(); + if (!Float.isFinite(scales[row])) { + throw new IllegalArgumentException(file + " has a non-finite scale for row " + row + + ": " + scales[row]); + } + } + final float[] decodedNorms = new float[rowCount]; + for (int row = 0; row < rowCount; row++) { + decodedNorms[row] = data.readFloat(); + if (!Float.isFinite(decodedNorms[row]) || decodedNorms[row] < 0) { + throw new IllegalArgumentException(file + " has an invalid decoded norm for row " + + row + ": " + decodedNorms[row]); + } + } + final int paddedDimension = HadamardRotation.paddedDimension(dimension); + final int rowBytes = (paddedDimension * bits + 7) / 8; + requireStorableSize(rowCount, rowBytes); + final byte[] codes = new byte[rowCount * rowBytes]; + try { + data.readFully(codes); + } catch (EOFException e) { + throw new IOException(file + " is truncated: the header declares " + rowCount + + " rows of " + rowBytes + " packed bytes, but the file ends early", e); + } + if (data.read() != -1) { + throw new IllegalArgumentException(file + " has trailing bytes after the declared " + + "content; it is not a quantized matrix of this version"); + } + return new QuantizedEmbeddingMatrix(rowCount, dimension, bits, seed, quantizer, scales, + codes, decodedNorms); + } + } + + /** + * {@return one packed code} Codes may straddle a byte boundary (3-bit widths do), so two + * adjacent bytes are read and the code's bits selected; the second byte is only touched when + * the code actually crosses into it, so the last code of a row never reads past its region. + * + * @param codes The packed code array. + * @param rowBase The row's first byte index. + * @param bits The code width. + * @param index The code's index within the row. + */ + private static int readCode(byte[] codes, int rowBase, int bits, int index) { + final int bitPosition = index * bits; + final int byteIndex = rowBase + (bitPosition >>> 3); + final int shift = bitPosition & 7; + int word = codes[byteIndex] & 0xFF; + if (shift + bits > 8) { + word |= (codes[byteIndex + 1] & 0xFF) << 8; + } + return (word >>> shift) & ((1 << bits) - 1); + } + + /** + * Writes one packed code, the mirror of {@link #readCode(byte[], int, int, int)}. + * + * @param codes The packed code array. + * @param rowBase The row's first byte index. + * @param bits The code width. + * @param index The code's index within the row. + * @param code The code value, within the bit width. + */ + private static void writeCode(byte[] codes, int rowBase, int bits, int index, int code) { + final int bitPosition = index * bits; + final int byteIndex = rowBase + (bitPosition >>> 3); + final int shift = bitPosition & 7; + codes[byteIndex] |= (byte) (code << shift); + if (shift + bits > 8) { + codes[byteIndex + 1] |= (byte) (code >>> (8 - shift)); + } + } + + /** + * {@return this row's code at an index, for tests} + * + * @param row The row. + * @param index The code index within the row, up to the padded dimension. + */ + int code(int row, int index) { + return readCode(codes, row * rowBytes, bits, index); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/GaussianQuantizerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/GaussianQuantizerTest.java new file mode 100644 index 0000000000..0484718dc3 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/GaussianQuantizerTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.Random; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The quantizer contract: the derived grids match the published Lloyd-Max tables for the + * Gaussian, encoding picks the nearest level, grids are symmetric, and a grid read back from a + * file round-trips through {@code fromLevels}. + */ +class GaussianQuantizerTest { + + @Test + void testGridsMatchThePublishedLloydMaxTables() { + // Reference values from Max, "Quantizing for minimum distortion", IRE Transactions on + // Information Theory 6(1), 1960, table for the standard normal: the positive levels of the + // symmetric optimal quantizer. The derivation here discretizes the density, so agreement is + // to the published tables' precision, not bit-exact. + assertPositiveLevels(GaussianQuantizer.forBits(2), 0.4528, 1.5104); + assertPositiveLevels(GaussianQuantizer.forBits(3), 0.2451, 0.7560, 1.3439, 2.1520); + assertPositiveLevels(GaussianQuantizer.forBits(4), + 0.1284, 0.3881, 0.6568, 0.9424, 1.2562, 1.6181, 2.0690, 2.7326); + } + + /** + * Asserts the upper half of a symmetric grid, and by symmetry the lower half. + * + * @param quantizer The quantizer under test. + * @param expected The published positive levels, ascending. + */ + private static void assertPositiveLevels(GaussianQuantizer quantizer, double... expected) { + final int half = quantizer.levelCount() / 2; + assertEquals(expected.length, half); + for (int i = 0; i < half; i++) { + assertEquals(expected[i], quantizer.level(half + i), 2e-3, + "positive level " + i + " must match the published Lloyd-Max table"); + assertEquals(-expected[i], quantizer.level(half - 1 - i), 2e-3, + "the grid must be symmetric"); + } + } + + @Test + void testEncodePicksTheNearestLevel() { + final Random random = new Random(42); + for (int bits = GaussianQuantizer.MIN_BITS; bits <= GaussianQuantizer.MAX_BITS; bits++) { + final GaussianQuantizer quantizer = GaussianQuantizer.forBits(bits); + for (int trial = 0; trial < 10_000; trial++) { + final float value = (float) (random.nextGaussian() * 2); + final int code = quantizer.encode(value); + final double encodedDistance = Math.abs(value - quantizer.level(code)); + for (int other = 0; other < quantizer.levelCount(); other++) { + assertTrue(encodedDistance <= Math.abs(value - quantizer.level(other)) + 1e-6, + "encode(" + value + ") chose level " + code + " but level " + other + + " is nearer"); + } + } + } + } + + @Test + void testEncodeCoversTheFullCodeRange() { + final GaussianQuantizer quantizer = GaussianQuantizer.forBits(2); + assertEquals(0, quantizer.encode(-10f)); + assertEquals(quantizer.levelCount() - 1, quantizer.encode(10f)); + } + + @Test + void testFromLevelsRoundTripsAGrid() { + final GaussianQuantizer original = GaussianQuantizer.forBits(3); + final GaussianQuantizer restored = GaussianQuantizer.fromLevels(original.levels()); + assertEquals(original.levelCount(), restored.levelCount()); + for (int code = 0; code < original.levelCount(); code++) { + assertEquals(original.level(code), restored.level(code), 0f); + } + final Random random = new Random(7); + for (int trial = 0; trial < 1_000; trial++) { + final float value = (float) (random.nextGaussian() * 2); + assertEquals(original.encode(value), restored.encode(value), + "a restored grid must encode exactly like its source"); + } + } + + @Test + void testFromLevelsRejectsMalformedGrids() { + assertThrows(IllegalArgumentException.class, () -> GaussianQuantizer.fromLevels(null)); + assertThrows(IllegalArgumentException.class, + () -> GaussianQuantizer.fromLevels(new float[] {1f, 2f, 3f})); + assertThrows(IllegalArgumentException.class, + () -> GaussianQuantizer.fromLevels(new float[] {-1f, -1f, 1f, 2f})); + assertThrows(IllegalArgumentException.class, + () -> GaussianQuantizer.fromLevels(new float[] {-1f, Float.NaN, 1f, 2f})); + assertThrows(IllegalArgumentException.class, + () -> GaussianQuantizer.fromLevels(new float[] {1f, 2f})); + } + + @Test + void testUnsupportedBitWidthsFailLoud() { + assertThrows(IllegalArgumentException.class, () -> GaussianQuantizer.forBits(1)); + assertThrows(IllegalArgumentException.class, () -> GaussianQuantizer.forBits(5)); + assertThrows(IllegalArgumentException.class, () -> GaussianQuantizer.forBits(0)); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java new file mode 100644 index 0000000000..d8c38de364 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.Arrays; +import java.util.Random; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The rotation contract: orthonormal (norms and dot products preserved), self-consistent + * (inverse restores the input), deterministic per seed and JVM-independent by construction, and + * padded to the next power of two. + */ +class HadamardRotationTest { + + @Test + void testPaddedDimensionIsTheNextPowerOfTwo() { + assertEquals(1, HadamardRotation.paddedDimension(1)); + assertEquals(2, HadamardRotation.paddedDimension(2)); + assertEquals(4, HadamardRotation.paddedDimension(3)); + assertEquals(256, HadamardRotation.paddedDimension(256)); + assertEquals(512, HadamardRotation.paddedDimension(300)); + assertEquals(1024, HadamardRotation.paddedDimension(1024)); + assertThrows(IllegalArgumentException.class, () -> HadamardRotation.paddedDimension(0)); + assertThrows(IllegalArgumentException.class, () -> HadamardRotation.paddedDimension(-5)); + } + + @Test + void testRotationPreservesNormsAndDotProducts() { + final Random random = new Random(42); + final HadamardRotation rotation = new HadamardRotation(300, 7L); + final float[] a = randomPadded(rotation, random); + final float[] b = randomPadded(rotation, random); + final double normBefore = norm(a); + final double dotBefore = dot(a, b); + rotation.rotate(a); + rotation.rotate(b); + assertEquals(normBefore, norm(a), 1e-3 * normBefore, + "an orthonormal transform preserves norms"); + assertEquals(dotBefore, dot(a, b), 1e-3 * (1 + Math.abs(dotBefore)), + "an orthonormal transform preserves dot products"); + } + + @Test + void testInverseRestoresTheInput() { + final Random random = new Random(43); + final HadamardRotation rotation = new HadamardRotation(256, 99L); + final float[] vector = randomPadded(rotation, random); + final float[] original = vector.clone(); + rotation.rotate(vector); + rotation.inverse(vector); + assertArrayEquals(original, vector, 1e-4f); + } + + @Test + void testSameSeedSameRotationDifferentSeedDifferentRotation() { + final Random random = new Random(44); + final float[] input = randomPadded(new HadamardRotation(64, 5L), random); + final float[] first = input.clone(); + final float[] second = input.clone(); + final float[] other = input.clone(); + new HadamardRotation(64, 5L).rotate(first); + new HadamardRotation(64, 5L).rotate(second); + new HadamardRotation(64, 6L).rotate(other); + assertArrayEquals(first, second, 0f, "the same seed must give bit-identical rotations"); + assertFalse(Arrays.equals(first, other), + "different seeds must give different rotations"); + } + + @Test + void testEnergySpreadsAcrossCoordinates() { + // A one-hot vector concentrates all its energy in one coordinate; after rotation every + // coordinate must hold a share, which is the property the per-coordinate quantizer needs. + final HadamardRotation rotation = new HadamardRotation(128, 11L); + final float[] oneHot = new float[rotation.paddedDimension()]; + oneHot[3] = 1f; + rotation.rotate(oneHot); + final double expectedMagnitude = 1.0 / Math.sqrt(rotation.paddedDimension()); + for (final float value : oneHot) { + assertEquals(expectedMagnitude, Math.abs(value), 1e-6, + "a rotated one-hot vector has equal magnitude everywhere"); + } + } + + @Test + void testRejectsWrongLengthAndNull() { + final HadamardRotation rotation = new HadamardRotation(300, 1L); + assertEquals(512, rotation.paddedDimension()); + assertThrows(IllegalArgumentException.class, () -> rotation.rotate(null)); + assertThrows(IllegalArgumentException.class, () -> rotation.rotate(new float[300])); + assertThrows(IllegalArgumentException.class, () -> rotation.inverse(new float[511])); + assertThrows(IllegalArgumentException.class, () -> new HadamardRotation(0, 1L)); + } + + @Test + void testDimensionOneIsTheIdentityUpToSign() { + final HadamardRotation rotation = new HadamardRotation(1, 123L); + final float[] vector = new float[] {2.5f}; + rotation.rotate(vector); + assertEquals(2.5f, Math.abs(vector[0]), 1e-6f); + rotation.inverse(vector); + assertEquals(2.5f, vector[0], 1e-6f); + } + + private static float[] randomPadded(HadamardRotation rotation, Random random) { + final float[] vector = new float[rotation.paddedDimension()]; + for (int i = 0; i < vector.length; i++) { + vector[i] = (float) random.nextGaussian(); + } + return vector; + } + + private static double norm(float[] vector) { + return Math.sqrt(dot(vector, vector)); + } + + private static double dot(float[] a, float[] b) { + double dot = 0; + for (int i = 0; i < a.length; i++) { + dot += (double) a[i] * b[i]; + } + return dot; + } + + @Test + void testHelpersKeepThePaddedContract() { + // The helpers above assume rotate() leaves length unchanged; pin that here. + final HadamardRotation rotation = new HadamardRotation(3, 0L); + final float[] vector = new float[] {1f, 2f, 3f, 0f}; + rotation.rotate(vector); + assertEquals(4, vector.length); + assertTrue(Float.isFinite(vector[0])); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java new file mode 100644 index 0000000000..fcb798f1bc --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Random; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The quantized matrix contract: reconstruction quality per bit width, rotated-space math that + * agrees with original-space math, linear pooling, deterministic bytes, a self-describing file + * that round-trips exactly, and loud failure on malformed input. + */ +class QuantizedEmbeddingMatrixTest { + + private static final int ROWS = 50; + private static final int DIMENSION = 300; + private static final long SEED = 12345L; + + /** + * {@return a deterministic random test matrix with varied row norms} + */ + private static float[] testMatrix() { + final Random random = new Random(42); + final float[] matrix = new float[ROWS * DIMENSION]; + for (int row = 0; row < ROWS; row++) { + // Vary the norms so per-row scaling is actually exercised. + final float rowScale = 0.1f + 3f * random.nextFloat(); + for (int d = 0; d < DIMENSION; d++) { + matrix[row * DIMENSION + d] = rowScale * (float) random.nextGaussian(); + } + } + return matrix; + } + + @Test + void testReconstructionQualityPerBitWidth() { + // The mean squared error of the Gaussian Lloyd-Max grids translates to an expected cosine + // between a row and its reconstruction; these thresholds sit safely below the analytic + // expectation (about 0.945 at 2 bits, 0.983 at 3, 0.995 at 4) but far above what a broken + // rotation, grid, or scale would produce. + assertMeanCosineAtLeast(2, 0.92); + assertMeanCosineAtLeast(3, 0.97); + assertMeanCosineAtLeast(4, 0.99); + } + + /** + * Asserts the mean cosine between original and decoded rows for a bit width. + * + * @param bits The bit width under test. + * @param threshold The minimum acceptable mean cosine. + */ + private static void assertMeanCosineAtLeast(int bits, double threshold) { + final float[] matrix = testMatrix(); + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, bits, SEED); + double cosineSum = 0; + for (int row = 0; row < ROWS; row++) { + final float[] decoded = quantized.decodeRow(row); + cosineSum += cosine(matrix, row * DIMENSION, decoded); + } + final double meanCosine = cosineSum / ROWS; + assertTrue(meanCosine >= threshold, bits + " bits reconstructed a mean cosine of " + + meanCosine + ", below the acceptable " + threshold); + } + + @Test + void testRotatedDotEqualsOriginalSpaceDot() { + final float[] matrix = testMatrix(); + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 4, SEED); + final Random random = new Random(7); + final float[] query = new float[DIMENSION]; + for (int d = 0; d < DIMENSION; d++) { + query[d] = (float) random.nextGaussian(); + } + final float[] rotatedQuery = quantized.rotate(query); + for (int row = 0; row < ROWS; row++) { + final float[] decoded = quantized.decodeRow(row); + double originalDot = 0; + for (int d = 0; d < DIMENSION; d++) { + originalDot += (double) decoded[d] * query[d]; + } + assertEquals(originalDot, quantized.dotRotated(row, rotatedQuery), + 1e-3 * (1 + Math.abs(originalDot)), + "the rotation is orthonormal, so rotated-space and original-space dots must agree"); + } + } + + @Test + void testRowNormIsTheDecodedRowsNorm() { + final float[] matrix = testMatrix(); + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 3, SEED); + for (int row = 0; row < ROWS; row++) { + final float[] decoded = quantized.decodeRow(row); + double sumOfSquares = 0; + for (final float value : decoded) { + sumOfSquares += (double) value * value; + } + final double decodedNorm = Math.sqrt(sumOfSquares); + assertEquals(decodedNorm, quantized.rowNorm(row), 1e-3 * (1 + decodedNorm)); + } + } + + @Test + void testPoolingInRotatedSpaceEqualsPoolingDecodedRows() { + final float[] matrix = testMatrix(); + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 4, SEED); + final float[] rotatedSum = new float[quantized.paddedDimension()]; + quantized.addRowRotated(0, 1f, rotatedSum); + quantized.addRowRotated(1, 2.5f, rotatedSum); + quantized.addRowRotated(2, -0.5f, rotatedSum); + final float[] pooled = quantized.toOriginal(rotatedSum); + final float[] row0 = quantized.decodeRow(0); + final float[] row1 = quantized.decodeRow(1); + final float[] row2 = quantized.decodeRow(2); + final float[] expected = new float[DIMENSION]; + for (int d = 0; d < DIMENSION; d++) { + expected[d] = row0[d] + 2.5f * row1[d] - 0.5f * row2[d]; + } + assertArrayEquals(expected, pooled, 1e-3f, + "rotation is linear, so pooling commutes with it"); + } + + @Test + void testZeroRowDecodesToZero() { + final float[] matrix = new float[3 * 8]; + matrix[0] = 1f; + matrix[2 * 8 + 5] = -2f; + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, 3, 8, 2, SEED); + assertArrayEquals(new float[8], quantized.decodeRow(1), 0f); + assertEquals(0.0, quantized.rowNorm(1), 0.0); + } + + @Test + void testQuantizingIsDeterministic(@TempDir Path directory) throws IOException { + final float[] matrix = testMatrix(); + final Path first = directory.resolve("first.bin"); + final Path second = directory.resolve("second.bin"); + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 3, SEED).write(first); + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 3, SEED).write(second); + assertArrayEquals(Files.readAllBytes(first), Files.readAllBytes(second), + "the same matrix, bits, and seed must produce the same file bytes"); + } + + @Test + void testWriteReadRoundTrip(@TempDir Path directory) throws IOException { + final float[] matrix = testMatrix(); + final QuantizedEmbeddingMatrix written = + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 4, SEED); + final Path file = directory.resolve("matrix.bin"); + written.write(file); + final QuantizedEmbeddingMatrix read = QuantizedEmbeddingMatrix.read(file); + assertEquals(written.rowCount(), read.rowCount()); + assertEquals(written.dimension(), read.dimension()); + assertEquals(written.paddedDimension(), read.paddedDimension()); + assertEquals(written.bits(), read.bits()); + assertEquals(written.seed(), read.seed()); + for (int row = 0; row < ROWS; row++) { + assertArrayEquals(written.decodeRow(row), read.decodeRow(row), 0f, + "a read matrix must decode exactly like the written one"); + } + // Writing the read matrix reproduces the file, so the format loses nothing. + final Path rewritten = directory.resolve("rewritten.bin"); + read.write(rewritten); + assertArrayEquals(Files.readAllBytes(file), Files.readAllBytes(rewritten)); + } + + @Test + void testQuantizedFileIsSmallerThanTheFloatMatrix(@TempDir Path directory) + throws IOException { + final float[] matrix = testMatrix(); + final Path file = directory.resolve("matrix.bin"); + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 4, SEED).write(file); + final long floatBytes = (long) ROWS * DIMENSION * Float.BYTES; + // 4 bits over the padded dimension (512 for 300) plus one scale per row and the header: + // still far under half the float size; at equal dimensions the ratio approaches 8x. + assertTrue(Files.size(file) < floatBytes / 2, + "the 4-bit file (" + Files.size(file) + " bytes) must be well under half the float " + + "matrix (" + floatBytes + " bytes)"); + } + + @Test + void testQuantizeValidatesItsArguments() { + final float[] matrix = new float[2 * 4]; + assertThrows(IllegalArgumentException.class, + () -> QuantizedEmbeddingMatrix.quantize(null, 2, 4, 2, SEED)); + assertThrows(IllegalArgumentException.class, + () -> QuantizedEmbeddingMatrix.quantize(matrix, 0, 4, 2, SEED)); + assertThrows(IllegalArgumentException.class, + () -> QuantizedEmbeddingMatrix.quantize(matrix, 2, 0, 2, SEED)); + assertThrows(IllegalArgumentException.class, + () -> QuantizedEmbeddingMatrix.quantize(matrix, 2, 5, 2, SEED)); + assertThrows(IllegalArgumentException.class, + () -> QuantizedEmbeddingMatrix.quantize(matrix, 2, 4, 1, SEED)); + assertThrows(IllegalArgumentException.class, + () -> QuantizedEmbeddingMatrix.quantize(matrix, 2, 4, 5, SEED)); + } + + @Test + void testQuantizeRejectsNonFiniteValuesNamingTheirPosition() { + final float[] matrix = new float[2 * 4]; + matrix[5] = Float.NaN; + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> QuantizedEmbeddingMatrix.quantize(matrix, 2, 4, 2, SEED)); + assertTrue(e.getMessage().contains("Row 1"), e.getMessage()); + assertTrue(e.getMessage().contains("dimension 1"), e.getMessage()); + } + + @Test + void testRotatedSpaceAccessorsValidate() { + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(new float[4 * 8], 4, 8, 2, SEED); + assertThrows(IllegalArgumentException.class, () -> quantized.decodeRow(-1)); + assertThrows(IllegalArgumentException.class, () -> quantized.decodeRow(4)); + assertThrows(IllegalArgumentException.class, () -> quantized.rotate(null)); + assertThrows(IllegalArgumentException.class, () -> quantized.rotate(new float[7])); + assertThrows(IllegalArgumentException.class, () -> quantized.toOriginal(new float[7])); + assertThrows(IllegalArgumentException.class, + () -> quantized.addRowRotated(0, 1f, new float[7])); + assertThrows(IllegalArgumentException.class, + () -> quantized.dotRotated(0, new float[7])); + } + + @Test + void testReadRejectsForeignAndTruncatedFiles(@TempDir Path directory) throws IOException { + final Path foreign = directory.resolve("foreign.bin"); + Files.write(foreign, new byte[] {1, 2, 3, 4, 5, 6, 7, 8}); + assertThrows(IllegalArgumentException.class, () -> QuantizedEmbeddingMatrix.read(foreign)); + + final Path file = directory.resolve("matrix.bin"); + QuantizedEmbeddingMatrix.quantize(testMatrix(), ROWS, DIMENSION, 2, SEED).write(file); + final byte[] full = Files.readAllBytes(file); + final Path truncated = directory.resolve("truncated.bin"); + Files.write(truncated, Arrays.copyOf(full, full.length - 10)); + assertThrows(IOException.class, () -> QuantizedEmbeddingMatrix.read(truncated)); + + final Path trailing = directory.resolve("trailing.bin"); + final byte[] extra = Arrays.copyOf(full, full.length + 1); + Files.write(trailing, extra); + assertThrows(IllegalArgumentException.class, () -> QuantizedEmbeddingMatrix.read(trailing)); + } + + /** + * {@return the cosine between a matrix row and a decoded vector} + * + * @param matrix The flat row-major matrix. + * @param base The row's first index. + * @param decoded The decoded row. + */ + private static double cosine(float[] matrix, int base, float[] decoded) { + double dot = 0; + double normASquared = 0; + double normBSquared = 0; + for (int d = 0; d < decoded.length; d++) { + dot += (double) matrix[base + d] * decoded[d]; + normASquared += (double) matrix[base + d] * matrix[base + d]; + normBSquared += (double) decoded[d] * decoded[d]; + } + return dot / (Math.sqrt(normASquared) * Math.sqrt(normBSquared)); + } +} From b5813b21779245c595749bece145bf9092843308 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 23 Jul 2026 23:51:56 -0400 Subject: [PATCH 72/82] OPENNLP-1895: Load quantized matrices in StaticEmbeddingModel and add the QuantizeModel tool --- .../opennlp/embeddings/EmbeddingTable.java | 89 ++++++ .../embeddings/FloatEmbeddingTable.java | 121 ++++++++ .../opennlp/embeddings/ModelFileNames.java | 6 + .../opennlp/embeddings/ModelQuantizer.java | 147 +++++++++ .../embeddings/QuantizedEmbeddingMatrix.java | 66 +++- .../embeddings/QuantizedTableAdapter.java | 77 +++++ .../embeddings/StaticEmbeddingModel.java | 287 +++++++++++++----- .../java/opennlp/embeddings/cmdline/CLI.java | 1 + .../cmdline/QuantizeModelParams.java | 51 ++++ .../embeddings/cmdline/QuantizeModelTool.java | 71 +++++ .../QuantizedEmbeddingMatrixTest.java | 48 +++ .../StaticEmbeddingModelQuantizedTest.java | 200 ++++++++++++ 12 files changed, 1079 insertions(+), 85 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelTool.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java new file mode 100644 index 0000000000..a4c89ec5b0 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +/** + * The row storage behind {@link StaticEmbeddingModel}: gathering rows into a pooled vector and + * scoring rows against a query, independent of whether the rows are float or quantized. + * + *

The seam is shaped so a storage form may pool in a working space of its own. + * {@link #addRow(int, float, float[])} accumulates into a vector of {@link #pooledLength()}, and + * {@link #finishPooling(float[])} maps the accumulated vector to original space once per pooled + * result. The float table's working space is original space and its finish is the identity; the + * quantized table pools in rotated space, where decoding a row is a grid lookup, and pays its + * single inverse rotation in the finish. Scoring mirrors this: {@link #prepareQuery(float[])} + * maps a query into the working space once, and {@link #dot(int, float[])} scores every row + * against the prepared query there.

+ * + *

Implementations are immutable and safe for concurrent use; the accumulator and prepared + * query arrays belong to the caller.

+ */ +interface EmbeddingTable { + + /** {@return the number of rows} */ + int rowCount(); + + /** {@return the original row width, the length of a pooled result} */ + int dimension(); + + /** {@return the length of the pooling accumulator and of a prepared query} */ + int pooledLength(); + + /** + * Adds a row, times a weight, onto a pooling accumulator. + * + * @param row The row to add, between 0 and {@code rowCount() - 1}. + * @param weight The weight to multiply the row by. + * @param sum The accumulator, of length {@link #pooledLength()}. + */ + void addRow(int row, float weight, float[] sum); + + /** + * Maps an accumulated vector to original space. Called once per pooled result; the returned + * array may be {@code sum} itself. + * + * @param sum The accumulator, of length {@link #pooledLength()}. + * @return The pooled vector in original space, of length {@link #dimension()}. + */ + float[] finishPooling(float[] sum); + + /** + * Maps an original-space query into this table's working space, once per scan. The returned + * array may be {@code query} itself. + * + * @param query The query, of length {@link #dimension()}. Not modified. + * @return The prepared query, of length {@link #pooledLength()}. + */ + float[] prepareQuery(float[] query); + + /** + * The dot product of a row with a prepared query, equal to the original-space dot product up + * to float rounding. + * + * @param row The row to score, between 0 and {@code rowCount() - 1}. + * @param preparedQuery The query as returned by {@link #prepareQuery(float[])}. + * @return The dot product. + */ + double dot(int row, float[] preparedQuery); + + /** + * {@return the L2 norm of a row as this table stores it, for cosine scoring} + * + * @param row The row, between 0 and {@code rowCount() - 1}. + */ + double rowNorm(int row); +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java new file mode 100644 index 0000000000..2230862707 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +/** + * The float {@link EmbeddingTable}: a flat row-major matrix, exactly the storage + * {@link StaticEmbeddingModel} always had. Its working space is original space, so query + * preparation and pooling finish are the identity, and per-row norms are precomputed once for + * the neighbor scan. + */ +final class FloatEmbeddingTable implements EmbeddingTable { + + private final float[] values; + private final int dimension; + private final int rowCount; + private final double[] rowNorms; + + /** + * Wraps a flat row-major matrix. The array is used as given; the loaders that construct this + * table own it exclusively. + * + * @param values The matrix, {@code rowCount * dimension} floats. + * @param dimension The row width. + * @param rowCount The number of rows. + */ + FloatEmbeddingTable(float[] values, int dimension, int rowCount) { + this.values = values; + this.dimension = dimension; + this.rowCount = rowCount; + this.rowNorms = new double[rowCount]; + for (int row = 0; row < rowCount; row++) { + final int base = row * dimension; + double sumOfSquares = 0; + for (int d = 0; d < dimension; d++) { + final float value = values[base + d]; + sumOfSquares += (double) value * value; + } + rowNorms[row] = Math.sqrt(sumOfSquares); + } + } + + @Override + public int rowCount() { + return rowCount; + } + + @Override + public int dimension() { + return dimension; + } + + @Override + public int pooledLength() { + return dimension; + } + + @Override + public void addRow(int row, float weight, float[] sum) { + final int base = row * dimension; + if (weight == 1f) { + for (int d = 0; d < dimension; d++) { + sum[d] += values[base + d]; + } + } else { + for (int d = 0; d < dimension; d++) { + sum[d] += values[base + d] * weight; + } + } + } + + @Override + public float[] finishPooling(float[] sum) { + return sum; + } + + @Override + public float[] prepareQuery(float[] query) { + return query; + } + + @Override + public double dot(int row, float[] preparedQuery) { + final int base = row * dimension; + // Four accumulators so the JIT can vectorize the dot product without reordering FP adds. + double dot0 = 0; + double dot1 = 0; + double dot2 = 0; + double dot3 = 0; + int d = 0; + for (final int limit = dimension - 3; d < limit; d += 4) { + dot0 += preparedQuery[d] * values[base + d]; + dot1 += preparedQuery[d + 1] * values[base + d + 1]; + dot2 += preparedQuery[d + 2] * values[base + d + 2]; + dot3 += preparedQuery[d + 3] * values[base + d + 3]; + } + double dot = dot0 + dot1 + dot2 + dot3; + for (; d < dimension; d++) { + dot += preparedQuery[d] * values[base + d]; + } + return dot; + } + + @Override + public double rowNorm(int row) { + return rowNorms[row]; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java index 657b814fc0..2a48d2c9b1 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java @@ -36,6 +36,12 @@ final class ModelFileNames { /** The safetensors file holding the embedding matrix and optional per-token weights. */ static final String SAFETENSORS = "model.safetensors"; + /** + * The quantized matrix file, written by the {@code QuantizeModel} tool. When present it wins + * over {@link #SAFETENSORS}: it carries the matrix and any per-token weights itself. + */ + static final String QUANTIZED = "model.quantized"; + /** The tokenizer description whose Unigram {@code model.vocab} order names the matrix rows. */ static final String TOKENIZER_JSON = "tokenizer.json"; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java new file mode 100644 index 0000000000..c440bf1b13 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Quantizes a static embedding model directory in place: reads the matrix and optional + * per-token weights from the directory's {@code model.safetensors}, quantizes the matrix to the + * requested bit width (see {@link QuantizedEmbeddingMatrix}), and writes + * {@code model.quantized} next to it. {@code StaticEmbeddingModel.load} prefers the quantized + * file from then on; the safetensors may be deleted for a slim deployment. + * + *

The written file is verified by reading it back and measuring the mean cosine between the + * original and reconstructed rows over a deterministic sample, so a completed run reports the + * reconstruction quality actually on disk.

+ */ +public final class ModelQuantizer { + + // At most this many rows enter the verification sample, evenly strided so it is + // deterministic and covers the whole table. + private static final int VERIFICATION_SAMPLE_CAP = 1024; + + /** Not instantiable. */ + private ModelQuantizer() { + } + + /** + * What a quantization run produced and measured. + * + * @param rowCount The number of matrix rows. + * @param dimension The row width. + * @param bits The bit width per padded dimension. + * @param hasWeights Whether per-token pooling weights were carried over. + * @param safetensorsBytes The size of the source safetensors file. + * @param quantizedBytes The size of the written quantized file. + * @param sampledRows The number of rows in the verification sample. + * @param meanCosine The mean cosine between original and reconstructed sampled rows; + * {@code Double.NaN} when every sampled row was zero. + */ + public record Result(int rowCount, int dimension, int bits, boolean hasWeights, + long safetensorsBytes, long quantizedBytes, int sampledRows, + double meanCosine) { + } + + /** + * Quantizes the model directory's matrix and writes {@code model.quantized}. + * + * @param modelDirectory The model directory. Must not be {@code null}, must be a directory, + * and must hold a {@code model.safetensors}. + * @param bits The bit width, between {@link QuantizedEmbeddingMatrix#MIN_BITS} and + * {@link QuantizedEmbeddingMatrix#MAX_BITS}. + * @param seed The rotation seed; the same matrix, bits, and seed write the same + * file bytes. + * @return What was produced and measured. + * @throws IllegalArgumentException Thrown if an argument is invalid or the directory has no + * safetensors file. + * @throws IOException Thrown if reading or writing fails. + */ + public static Result quantize(Path modelDirectory, int bits, long seed) throws IOException { + if (modelDirectory == null) { + throw new IllegalArgumentException("ModelDirectory must not be null"); + } + if (!Files.isDirectory(modelDirectory)) { + throw new IllegalArgumentException( + "Model directory does not exist or is not a directory: " + modelDirectory); + } + final Path safetensorsFile = modelDirectory.resolve(ModelFileNames.SAFETENSORS); + if (!Files.isRegularFile(safetensorsFile)) { + throw new IllegalArgumentException("Model directory " + modelDirectory + " has no " + + ModelFileNames.SAFETENSORS + " to quantize"); + } + final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); + final String matrixName = tensors.singleMatrixTensorName(); + final TensorInfo matrixInfo = tensors.tensorInfo(matrixName); + final int rowCount = matrixInfo.shape()[0]; + final int dimension = matrixInfo.shape()[1]; + final float[] matrix = tensors.readFloats(matrixName); + float[] weights = null; + if (tensors.tensorNames().contains("weights")) { + weights = tensors.readFloats("weights"); + if (weights.length != rowCount) { + throw new IllegalArgumentException("Tensor 'weights' in " + safetensorsFile + " has " + + weights.length + " elements but the matrix has " + rowCount + " rows"); + } + } + final Path quantizedFile = modelDirectory.resolve(ModelFileNames.QUANTIZED); + QuantizedEmbeddingMatrix.quantize(matrix, rowCount, dimension, bits, seed) + .withPoolingWeights(weights) + .write(quantizedFile); + // Verify what is actually on disk, not the in-memory object. + final QuantizedEmbeddingMatrix written = QuantizedEmbeddingMatrix.read(quantizedFile); + final int stride = Math.max(1, rowCount / VERIFICATION_SAMPLE_CAP); + int sampled = 0; + int nonZero = 0; + double cosineSum = 0; + for (int row = 0; row < rowCount; row += stride) { + sampled++; + final double cosine = cosine(matrix, row * dimension, dimension, written.decodeRow(row)); + if (!Double.isNaN(cosine)) { + nonZero++; + cosineSum += cosine; + } + } + return new Result(rowCount, dimension, bits, weights != null, + Files.size(safetensorsFile), Files.size(quantizedFile), sampled, + nonZero == 0 ? Double.NaN : cosineSum / nonZero); + } + + /** + * {@return the cosine between a matrix row and its reconstruction, or {@code Double.NaN} when + * either has no direction} + * + * @param matrix The flat row-major matrix. + * @param base The row's first index. + * @param dimension The row width. + * @param decoded The reconstructed row. + */ + private static double cosine(float[] matrix, int base, int dimension, float[] decoded) { + double dot = 0; + double normASquared = 0; + double normBSquared = 0; + for (int d = 0; d < dimension; d++) { + dot += (double) matrix[base + d] * decoded[d]; + normASquared += (double) matrix[base + d] * matrix[base + d]; + normBSquared += (double) decoded[d] * decoded[d]; + } + final double denominator = Math.sqrt(normASquared) * Math.sqrt(normBSquared); + return denominator == 0 ? Double.NaN : dot / denominator; + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java index 9e9a6595ee..cbddaaa56a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java @@ -91,13 +91,17 @@ public final class QuantizedEmbeddingMatrix { // time (one inverse rotation per row) and stored in the file rather than recomputed from the // codes on load. private final float[] decodedNorms; + // Optional per-row pooling weights carried alongside the matrix, so a quantized file can + // fully replace a safetensors file that bundled a "weights" tensor; null when absent. The + // weights are stored as they are, not quantized. + private final float[] poolingWeights; /** * Holds validated state; callers reach this through {@link #quantize} or {@link #read}. */ private QuantizedEmbeddingMatrix(int rowCount, int dimension, int bits, long seed, GaussianQuantizer quantizer, float[] scales, byte[] codes, - float[] decodedNorms) { + float[] decodedNorms, float[] poolingWeights) { this.rowCount = rowCount; this.dimension = dimension; this.paddedDimension = HadamardRotation.paddedDimension(dimension); @@ -109,6 +113,7 @@ private QuantizedEmbeddingMatrix(int rowCount, int dimension, int bits, long see this.scales = scales; this.codes = codes; this.decodedNorms = decodedNorms; + this.poolingWeights = poolingWeights; } /** @@ -208,7 +213,45 @@ public static QuantizedEmbeddingMatrix quantize(float[] rowMajor, int rowCount, decodedNorms[row] = (float) Math.sqrt(decodedSumOfSquares); } return new QuantizedEmbeddingMatrix(rowCount, dimension, bits, seed, quantizer, scales, - codes, decodedNorms); + codes, decodedNorms, null); + } + + /** + * {@return a copy of this matrix carrying per-row pooling weights} The weights ride along in + * the file unquantized, so a quantized file can fully replace a safetensors file that bundled + * a {@code weights} tensor. + * + * @param weights One weight per row, or {@code null} to carry none. Every weight must be + * finite. The array is copied. + * @return A matrix sharing this one's codes and scales, with the given weights. + * @throws IllegalArgumentException Thrown if {@code weights} has the wrong length or a + * non-finite value. + */ + public QuantizedEmbeddingMatrix withPoolingWeights(float[] weights) { + if (weights == null) { + return new QuantizedEmbeddingMatrix(rowCount, dimension, bits, seed, quantizer, scales, + codes, decodedNorms, null); + } + if (weights.length != rowCount) { + throw new IllegalArgumentException("Weights has " + weights.length + " values but this " + + "matrix has " + rowCount + " rows"); + } + for (int row = 0; row < rowCount; row++) { + if (!Float.isFinite(weights[row])) { + throw new IllegalArgumentException("Weight for row " + row + " is not finite: " + + weights[row]); + } + } + return new QuantizedEmbeddingMatrix(rowCount, dimension, bits, seed, quantizer, scales, + codes, decodedNorms, Arrays.copyOf(weights, weights.length)); + } + + /** + * {@return a copy of the per-row pooling weights, or {@code null} when this matrix carries + * none} + */ + public float[] poolingWeights() { + return poolingWeights == null ? null : Arrays.copyOf(poolingWeights, poolingWeights.length); } /** @@ -429,6 +472,12 @@ public void write(Path file) throws IOException { for (final float decodedNorm : decodedNorms) { data.writeFloat(decodedNorm); } + data.writeBoolean(poolingWeights != null); + if (poolingWeights != null) { + for (final float weight : poolingWeights) { + data.writeFloat(weight); + } + } data.write(codes); } } @@ -494,6 +543,17 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { + row + ": " + decodedNorms[row]); } } + float[] poolingWeights = null; + if (data.readBoolean()) { + poolingWeights = new float[rowCount]; + for (int row = 0; row < rowCount; row++) { + poolingWeights[row] = data.readFloat(); + if (!Float.isFinite(poolingWeights[row])) { + throw new IllegalArgumentException(file + " has a non-finite pooling weight for " + + "row " + row + ": " + poolingWeights[row]); + } + } + } final int paddedDimension = HadamardRotation.paddedDimension(dimension); final int rowBytes = (paddedDimension * bits + 7) / 8; requireStorableSize(rowCount, rowBytes); @@ -509,7 +569,7 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { + "content; it is not a quantized matrix of this version"); } return new QuantizedEmbeddingMatrix(rowCount, dimension, bits, seed, quantizer, scales, - codes, decodedNorms); + codes, decodedNorms, poolingWeights); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java new file mode 100644 index 0000000000..868ec39878 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +/** + * The quantized {@link EmbeddingTable}: a {@link QuantizedEmbeddingMatrix} whose working space + * is rotated space, so pooling accumulates decoded rows there and pays one inverse rotation per + * pooled result, and a scan rotates the query once and scores every row without leaving rotated + * space (see the matrix's class comment for why both are safe). + */ +final class QuantizedTableAdapter implements EmbeddingTable { + + private final QuantizedEmbeddingMatrix matrix; + + /** + * Wraps a quantized matrix. + * + * @param matrix The matrix to serve rows from. + */ + QuantizedTableAdapter(QuantizedEmbeddingMatrix matrix) { + this.matrix = matrix; + } + + @Override + public int rowCount() { + return matrix.rowCount(); + } + + @Override + public int dimension() { + return matrix.dimension(); + } + + @Override + public int pooledLength() { + return matrix.paddedDimension(); + } + + @Override + public void addRow(int row, float weight, float[] sum) { + matrix.addRowRotated(row, weight, sum); + } + + @Override + public float[] finishPooling(float[] sum) { + return matrix.toOriginal(sum); + } + + @Override + public float[] prepareQuery(float[] query) { + return matrix.rotate(query); + } + + @Override + public double dot(int row, float[] preparedQuery) { + return matrix.dotRotated(row, preparedQuery); + } + + @Override + public double rowNorm(int row) { + return matrix.rowNorm(row); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index c521d5049c..5afa3d3f5c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -59,6 +59,15 @@ * count of pooled pieces, not the sum of weights. A text with no in-vocabulary pieces yields a * zero vector.

* + *

Either layout may carry its matrix quantized in a {@code model.quantized} file (written by + * the {@code QuantizeModel} tool), which holds the matrix and any per-token pooling weights + * itself. A directory presents exactly one matrix file: the quantized file or the safetensors, + * not both. A directory carrying both is rejected, because the quantizer writes the quantized + * file next to the safetensors it read and so a directory holding both has not declared which + * is authoritative; delete one to choose. Embedding and similarity over a quantized matrix + * behave identically up to the quantization error of the chosen bit width; see + * {@link QuantizedEmbeddingMatrix} for the storage and its cost.

+ * *

A model directory may additionally carry a {@code terms.txt}: whole words and multi-word * phrases distilled through the teacher as units, owning the matrix rows after the subword rows * (see {@link ModelDistiller}). Embedding then matches the text against these terms greedily @@ -95,7 +104,8 @@ public enum Normalization { } private static final float NORMALIZE_EPSILON = 1e-12f; - private static final String WEIGHTS_TENSOR_NAME = "weights"; + // Shared with ModelQuantizer, which carries this tensor into the quantized file. + static final String WEIGHTS_TENSOR_NAME = "weights"; // The only pooling this model implements; the value the distiller writes into config.json. private static final String MEAN_POOLING = "mean"; private static final int[] NO_EXCLUDED_ROWS = new int[0]; @@ -106,7 +116,7 @@ public enum Normalization { private static final Set SENTENCEPIECE_SPECIAL_TOKENS = Set.of("", "", "", "", ""); - private final float[] embeddings; + private final EmbeddingTable table; private final float[] weights; private final int dimension; private final EmbeddingVocabulary vocabulary; @@ -114,29 +124,60 @@ public enum Normalization { // Tokenizer-id test for pieces that are never pooled (delimiter, control, unknown pieces). private final IntPredicate skipPieceId; private final boolean normalize; - // Per-row L2 norms and special-token mask, precomputed at load time for the neighbor scan. - private final double[] rowNorms; + // Special-token mask, precomputed at load time for the neighbor scan. private final boolean[] specialRows; // The term rows after the subword rows; empty for a model without a term table. private final TermTable terms; /** Holds the loaded, validated state; callers reach this through the {@code load} factories. */ - private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, + private StaticEmbeddingModel(EmbeddingTable table, float[] weights, EmbeddingVocabulary vocabulary, SubwordTokenizer tokenizer, - IntPredicate skipPieceId, boolean normalize, double[] rowNorms, + IntPredicate skipPieceId, boolean normalize, boolean[] specialRows, TermTable terms) { - this.embeddings = embeddings; + this.table = table; this.weights = weights; - this.dimension = dimension; + this.dimension = table.dimension(); this.vocabulary = vocabulary; this.tokenizer = tokenizer; this.skipPieceId = skipPieceId; this.normalize = normalize; - this.rowNorms = rowNorms; this.specialRows = specialRows; this.terms = terms; } + /** An embedding table and the optional per-token pooling weights that came with it. */ + private record TableAndWeights(EmbeddingTable table, float[] weights) { + } + + /** + * Reads a quantized table, holding its row count to the vocabulary's size plus the term + * count. + * + * @param quantizedFile The quantized matrix file. + * @param vocabulary The matrix row vocabulary. + * @param termCount The number of term rows after the vocabulary rows. + * @param vocabularySourceName The vocabulary's source, for error messages. + * @return The table and the pooling weights the file carries, if any. + * @throws InvalidFormatException Thrown if the row count disagrees with the vocabulary. + * @throws IOException Thrown if reading the file fails. + */ + private static TableAndWeights readQuantizedTable(Path quantizedFile, + EmbeddingVocabulary vocabulary, + int termCount, + String vocabularySourceName) + throws IOException { + final QuantizedEmbeddingMatrix matrix = QuantizedEmbeddingMatrix.read(quantizedFile); + final int expectedRows = vocabulary.size() + termCount; + if (matrix.rowCount() != expectedRows) { + throw new InvalidFormatException("Vocabulary " + vocabularySourceName + " has " + + vocabulary.size() + " tokens" + + (termCount > 0 ? " plus " + termCount + " terms" : "") + + " but quantized matrix " + quantizedFile + " has " + + matrix.rowCount() + " rows; these files do not belong to the same model"); + } + return new TableAndWeights(new QuantizedTableAdapter(matrix), matrix.poolingWeights()); + } + /** * Loads a static embedding model from a model directory, detecting the tokenizer family from * the files present and reading the pooling switch ({@code normalize}) from the model's @@ -155,6 +196,11 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, * is a SentencePiece model; the {@code .model} file carries its own text normalizer, so there * is no casing switch to read.

* + *

In either layout, the matrix comes from a {@code model.quantized} file when the directory + * has one and no {@code model.safetensors}; a directory holding both is rejected (see the + * class comment). After quantizing, delete the safetensors to deploy the quantized matrix, or + * delete the quantized file to fall back to the float matrix.

+ * * @param modelDirectory The model directory. Must not be {@code null} and must be a * directory. * @return The loaded model. @@ -185,9 +231,15 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException ModelFileNames.SENTENCEPIECE_MODELS); final Path tokenizerJsonFile = modelDirectory.resolve(ModelFileNames.TOKENIZER_JSON); if (sentencePieceModelFile != null && Files.isRegularFile(tokenizerJsonFile)) { + final Normalization normalization = + requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG)); + final Path quantizedFile = quantizedMatrixFileOrNull(modelDirectory); + if (quantizedFile != null) { + return loadSentencePieceQuantized(sentencePieceModelFile, tokenizerJsonFile, + quantizedFile, normalization, termLines, termsFile.toString()); + } return loadSentencePiece(sentencePieceModelFile, tokenizerJsonFile, - requiredFile(modelDirectory, ModelFileNames.SAFETENSORS), - requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG)), + requiredFile(modelDirectory, ModelFileNames.SAFETENSORS), normalization, termLines, termsFile.toString()); } if (Files.isRegularFile(tokenizerJsonFile)) { @@ -218,7 +270,6 @@ private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, List termLines, String termsSourceName) throws IOException { - final Path safetensorsFile = requiredFile(modelDirectory, ModelFileNames.SAFETENSORS); final Path tokenizerConfigFile = requiredFile(modelDirectory, ModelFileNames.TOKENIZER_CONFIG); final Normalization normalization = @@ -239,8 +290,44 @@ private static StaticEmbeddingModel loadWordpieceDirectory(Path modelDirectory, + "with load(vocabularyFile, safetensorsFile, casing, normalization) after choosing " + "deliberately"); } - return loadWordpiece(vocabularyFile, safetensorsFile, - lowerCase ? Casing.UNCASED : Casing.CASED, normalization, termLines, termsSourceName); + final Casing casing = lowerCase ? Casing.UNCASED : Casing.CASED; + final Path quantizedFile = quantizedMatrixFileOrNull(modelDirectory); + if (quantizedFile != null) { + final EmbeddingVocabulary vocabulary = EmbeddingVocabulary.fromVocabTxt(vocabularyFile); + final TermTable terms = TermTable.of(termLines, vocabulary.size(), termsSourceName); + return createWordpiece(vocabulary, + readQuantizedTable(quantizedFile, vocabulary, terms.size(), + vocabularyFile.toString()), + casing, normalization, vocabularyFile.toString(), terms); + } + return loadWordpiece(vocabularyFile, + requiredFile(modelDirectory, ModelFileNames.SAFETENSORS), + casing, normalization, termLines, termsSourceName); + } + + /** + * Resolves which matrix file a model directory presents, failing loud when the choice is + * ambiguous. The quantizer writes {@code model.quantized} next to the {@code model.safetensors} + * it read, so a directory holding both has not declared which is authoritative; deleting one + * makes the deployment's choice explicit rather than letting the loader guess. + * + * @param modelDirectory The model directory. + * @return the {@code model.quantized} file when it is the directory's only matrix file, or + * {@code null} when the directory presents only a {@code model.safetensors}. + * @throws InvalidFormatException Thrown if the directory holds both matrix files. + */ + private static Path quantizedMatrixFileOrNull(Path modelDirectory) throws InvalidFormatException { + final Path quantizedFile = modelDirectory.resolve(ModelFileNames.QUANTIZED); + if (!Files.isRegularFile(quantizedFile)) { + return null; + } + if (Files.isRegularFile(modelDirectory.resolve(ModelFileNames.SAFETENSORS))) { + throw new InvalidFormatException("Model directory " + modelDirectory + " has both " + + ModelFileNames.QUANTIZED + " and " + ModelFileNames.SAFETENSORS + "; delete one so " + + "the matrix source is unambiguous (keep " + ModelFileNames.QUANTIZED + " for a " + + "quantized deployment, or " + ModelFileNames.SAFETENSORS + " for the float matrix)"); + } + return quantizedFile; } /** @@ -355,9 +442,36 @@ private static StaticEmbeddingModel loadWordpiece(Path vocabularyFile, Path safe final TermTable terms = TermTable.of(termLines, vocabulary.size(), termsSourceName); final Matrix matrix = readMatrix(vocabulary, terms.size(), safetensorsFile, vocabularyFile.toString()); + final TableAndWeights tableAndWeights = new TableAndWeights( + new FloatEmbeddingTable(matrix.embeddings(), matrix.dimension(), + vocabulary.size() + terms.size()), + matrix.weights()); + return createWordpiece(vocabulary, tableAndWeights, casing, normalization, + vocabularyFile.toString(), terms); + } + + /** + * Builds a WordPiece model over a loaded table, whatever its storage form. + * + * @param vocabulary The matrix row vocabulary. + * @param tableAndWeights The table and its optional pooling weights. + * @param casing The tokenizer casing. + * @param normalization The pooling normalization. + * @param vocabularySourceName The vocabulary's source, for error messages. + * @param terms The term rows after the subword rows; empty for none. + * @return The loaded model. + * @throws InvalidFormatException Thrown if the vocabulary has no unknown token. + */ + private static StaticEmbeddingModel createWordpiece(EmbeddingVocabulary vocabulary, + TableAndWeights tableAndWeights, + Casing casing, + Normalization normalization, + String vocabularySourceName, + TermTable terms) + throws InvalidFormatException { final int unknownId = vocabulary.id(WordpieceTokenizer.BERT_UNK_TOKEN); if (unknownId < 0) { - throw new InvalidFormatException("Vocabulary " + vocabularyFile + " has no " + throw new InvalidFormatException("Vocabulary " + vocabularySourceName + " has no " + WordpieceTokenizer.BERT_UNK_TOKEN + " token; a WordPiece embedding model needs an " + "unknown token as the fallback for out-of-vocabulary text"); } @@ -369,10 +483,9 @@ private static StaticEmbeddingModel loadWordpiece(Path vocabularyFile, Path safe final int separatorId = vocabulary.id(WordpieceTokenizer.BERT_SEP_TOKEN); final IntPredicate skipPieceId = id -> id == unknownId || id == classificationId || id == separatorId; - return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), + return new StaticEmbeddingModel(tableAndWeights.table(), tableAndWeights.weights(), vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, - rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size() + terms.size()), - specialRows(vocabulary, WORDPIECE_SPECIAL_TOKENS, vocabulary.size() + terms.size()), + specialRows(vocabulary, WORDPIECE_SPECIAL_TOKENS, tableAndWeights.table().rowCount()), terms); } @@ -489,13 +602,65 @@ private static StaticEmbeddingModel loadSentencePiece(Path sentencePieceModelFil requireVocabularyCoverage(tokenizer, vocabulary, sentencePieceModelFile, tokenizerJsonFile); final Matrix matrix = readMatrix(vocabulary, terms.size(), safetensorsFile, tokenizerJsonFile.toString()); + final TableAndWeights tableAndWeights = new TableAndWeights( + new FloatEmbeddingTable(matrix.embeddings(), matrix.dimension(), + vocabulary.size() + terms.size()), + matrix.weights()); + return createSentencePiece(tokenizer, vocabulary, tableAndWeights, normalization, terms); + } + + /** + * Loads the SentencePiece layout over a quantized matrix file. + * + * @param sentencePieceModelFile The trained SentencePiece {@code .model} file. + * @param tokenizerJsonFile The Unigram {@code tokenizer.json} naming the matrix rows. + * @param quantizedFile The quantized matrix file. + * @param normalization The pooling normalization. + * @param termLines The terms in row order; empty without a term table. + * @param termsSourceName The terms' source, for error messages. + * @return The loaded model. + * @throws IOException Thrown if reading a file fails. + */ + private static StaticEmbeddingModel loadSentencePieceQuantized(Path sentencePieceModelFile, + Path tokenizerJsonFile, + Path quantizedFile, + Normalization normalization, + List termLines, + String termsSourceName) + throws IOException { + final EmbeddingVocabulary vocabulary = + EmbeddingVocabulary.fromTokenizerJson(tokenizerJsonFile); + final TermTable terms = TermTable.of(termLines, vocabulary.size(), termsSourceName); + final SentencePieceTokenizer tokenizer = + SentencePieceTokenizer.load(sentencePieceModelFile); + requireVocabularyCoverage(tokenizer, vocabulary, sentencePieceModelFile, tokenizerJsonFile); + return createSentencePiece(tokenizer, vocabulary, + readQuantizedTable(quantizedFile, vocabulary, terms.size(), + tokenizerJsonFile.toString()), + normalization, terms); + } + + /** + * Builds a SentencePiece model over a loaded table, whatever its storage form. + * + * @param tokenizer The loaded SentencePiece tokenizer. + * @param vocabulary The matrix row vocabulary. + * @param tableAndWeights The table and its optional pooling weights. + * @param normalization The pooling normalization. + * @param terms The term rows after the subword rows; empty for none. + * @return The loaded model. + */ + private static StaticEmbeddingModel createSentencePiece(SentencePieceTokenizer tokenizer, + EmbeddingVocabulary vocabulary, + TableAndWeights tableAndWeights, + Normalization normalization, + TermTable terms) { final IntPredicate skipPieceId = id -> tokenizer.isUnknown(id) || tokenizer.isControl(id); - return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), + return new StaticEmbeddingModel(tableAndWeights.table(), tableAndWeights.weights(), vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, - rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size() + terms.size()), specialRows(vocabulary, SENTENCEPIECE_SPECIAL_TOKENS, - vocabulary.size() + terms.size()), + tableAndWeights.table().rowCount()), terms); } @@ -595,27 +760,6 @@ private static Matrix readMatrix(EmbeddingVocabulary vocabulary, int termCount, return new Matrix(embeddings, weights, dimension); } - /** - * {@return the L2 norm of every matrix row, precomputed for the neighbor scan} - * - * @param embeddings The flat row-major matrix. - * @param dimension The row width. - * @param rowCount The number of rows. - */ - private static double[] rowNorms(float[] embeddings, int dimension, int rowCount) { - final double[] rowNorms = new double[rowCount]; - for (int row = 0; row < rowCount; row++) { - final int base = row * dimension; - double sumOfSquares = 0; - for (int d = 0; d < dimension; d++) { - final float value = embeddings[base + d]; - sumOfSquares += (double) value * value; - } - rowNorms[row] = Math.sqrt(sumOfSquares); - } - return rowNorms; - } - /** * {@return the mask of rows holding special tokens, excluded from neighbor results; term rows * are never special} @@ -664,38 +808,31 @@ public float[] embed(String text) { if (text == null) { throw new IllegalArgumentException("Text must not be null"); } - final float[] sum = new float[dimension]; + // Pooling accumulates in the table's working space (original space for the float table, + // rotated space for the quantized one) and maps to original space once per text. + final float[] sum = new float[table.pooledLength()]; // The count travels through the IntConsumer as a one-element array. - final int[] pooled = new int[1]; + final int[] pooledCount = new int[1]; forEachPooledRow(text, row -> { - final int base = row * dimension; - if (weights == null) { - for (int d = 0; d < dimension; d++) { - sum[d] += embeddings[base + d]; - } - } else { - final float weight = weights[row]; - for (int d = 0; d < dimension; d++) { - sum[d] += embeddings[base + d] * weight; - } - } - pooled[0]++; + table.addRow(row, weights == null ? 1f : weights[row], sum); + pooledCount[0]++; }); - final int denominator = Math.max(pooled[0], 1); - for (int d = 0; d < dimension; d++) { - sum[d] /= denominator; + final int denominator = Math.max(pooledCount[0], 1); + for (int i = 0; i < sum.length; i++) { + sum[i] /= denominator; } + final float[] pooled = table.finishPooling(sum); if (normalize) { double sumOfSquares = 0; - for (final float value : sum) { + for (final float value : pooled) { sumOfSquares += (double) value * value; } final float norm = (float) Math.max(Math.sqrt(sumOfSquares), NORMALIZE_EPSILON); for (int d = 0; d < dimension; d++) { - sum[d] /= norm; + pooled[d] /= norm; } } - return sum; + return pooled; } /** @@ -890,7 +1027,10 @@ private List nearestNeighbors(float[] query, int topK, int[] sortedExc if (queryNorm < NORMALIZE_EPSILON) { return List.of(); } - final int rowCount = rowNorms.length; + // The query maps into the table's working space once; every row is scored there. Norms are + // unchanged by the mapping, so the cosine denominator uses the original query norm. + final float[] preparedQuery = table.prepareQuery(query); + final int rowCount = table.rowCount(); // The capacity sizes the candidate arrays; a topK beyond the vocabulary (the scan can never // yield more than every row) would otherwise allocate topK-sized arrays or overflow. final TopK best = new TopK(Math.min(topK, rowCount)); @@ -903,30 +1043,13 @@ private List nearestNeighbors(float[] query, int topK, int[] sortedExc if (specialRows[row]) { continue; } - final double rowNorm = rowNorms[row]; + final double rowNorm = table.rowNorm(row); if (rowNorm < NORMALIZE_EPSILON) { // A zero row has no direction; scored 0 rather than NaN from a 0/0 division. best.offer(row, 0.0); continue; } - final int base = row * dimension; - // Four accumulators so the JIT can vectorize the dot product without reordering FP adds. - double dot0 = 0; - double dot1 = 0; - double dot2 = 0; - double dot3 = 0; - int d = 0; - for (final int limit = dimension - 3; d < limit; d += 4) { - dot0 += query[d] * embeddings[base + d]; - dot1 += query[d + 1] * embeddings[base + d + 1]; - dot2 += query[d + 2] * embeddings[base + d + 2]; - dot3 += query[d + 3] * embeddings[base + d + 3]; - } - double dot = dot0 + dot1 + dot2 + dot3; - for (; d < dimension; d++) { - dot += query[d] * embeddings[base + d]; - } - best.offer(row, dot / (queryNorm * rowNorm)); + best.offer(row, table.dot(row, preparedQuery) / (queryNorm * rowNorm)); } final Neighbor[] ordered = new Neighbor[best.size()]; for (int i = ordered.length - 1; i >= 0; i--) { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java index 50d0c42189..efd4d11717 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/CLI.java @@ -49,6 +49,7 @@ public final class CLI { tools.add(new AssembleModelTool()); tools.add(new DistillModelTool()); + tools.add(new QuantizeModelTool()); for (CmdLineTool tool : tools) { toolLookupMap.put(tool.getName(), tool); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java new file mode 100644 index 0000000000..ab79d9ba1c --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings.cmdline; + +import java.io.File; + +import opennlp.tools.cmdline.ArgumentParser.OptionalParameter; +import opennlp.tools.cmdline.ArgumentParser.ParameterDescription; + +/** + * The command-line arguments of {@link QuantizeModelTool}. + */ +interface QuantizeModelParams { + + /** + * {@return the model directory whose safetensors matrix is quantized in place} + */ + @ParameterDescription(valueName = "dir", + description = "the model directory whose model.safetensors is quantized in place") + File getModelDir(); + + /** + * {@return the bit width per dimension} + */ + @ParameterDescription(valueName = "bits", + description = "bits per dimension, 2 to 4; fewer bits, smaller file, lower fidelity") + @OptionalParameter(defaultValue = "4") + Integer getBits(); + + /** + * {@return the rotation seed; the same matrix, bits, and seed write the same file} + */ + @ParameterDescription(valueName = "seed", + description = "the rotation seed; the same matrix, bits, and seed write the same file") + @OptionalParameter(defaultValue = "0") + Integer getSeed(); +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelTool.java new file mode 100644 index 0000000000..8b83861e9d --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelTool.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings.cmdline; + +import java.io.IOException; +import java.util.Locale; + +import opennlp.embeddings.ModelQuantizer; +import opennlp.tools.cmdline.BasicCmdLineTool; +import opennlp.tools.cmdline.TerminateToolException; + +/** + * Quantizes a static embedding model directory's matrix to 2-4 bits per dimension, writing + * {@code model.quantized} next to the {@code model.safetensors}, and prints the sizes and the + * measured reconstruction quality. {@code StaticEmbeddingModel.load} prefers the quantized file + * from then on; the safetensors may be deleted for a slim deployment. + */ +public class QuantizeModelTool extends BasicCmdLineTool { + + interface Params extends QuantizeModelParams { + } + + @Override + public String getShortDescription() { + return "Quantizes a static embedding model directory's matrix to 2-4 bits per dimension"; + } + + @Override + public String getHelp() { + return getBasicHelp(Params.class); + } + + @Override + public void run(String[] args) { + final Params params = validateAndParseParams(args, Params.class); + final ModelQuantizer.Result result; + try { + result = ModelQuantizer.quantize(params.getModelDir().toPath(), params.getBits(), + params.getSeed()); + } catch (IllegalArgumentException e) { + throw new TerminateToolException(1, e.getMessage()); + } catch (IOException e) { + throw new TerminateToolException(-1, + "IO error while quantizing " + params.getModelDir() + ": " + e.getMessage(), e); + } + System.out.println("Quantized " + result.rowCount() + " rows of dimension " + + result.dimension() + " to " + result.bits() + " bits" + + (result.hasWeights() ? ", carrying the per-token weights" : "")); + System.out.println(String.format(Locale.ROOT, + "Size: %,d bytes safetensors, %,d bytes quantized (%.1fx smaller)", + result.safetensorsBytes(), result.quantizedBytes(), + result.safetensorsBytes() / (double) result.quantizedBytes())); + System.out.println(String.format(Locale.ROOT, + "Verified from disk: mean cosine %.4f between original and reconstructed rows " + + "(%d sampled)", result.meanCosine(), result.sampledRows())); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java index fcb798f1bc..f7304299ca 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java @@ -207,6 +207,54 @@ void testQuantizedFileIsSmallerThanTheFloatMatrix(@TempDir Path directory) + "matrix (" + floatBytes + " bytes)"); } + @Test + void testPoolingWeightsRoundTripThroughTheFile(@TempDir Path directory) throws IOException { + final float[] matrix = testMatrix(); + final float[] weights = new float[ROWS]; + for (int row = 0; row < ROWS; row++) { + weights[row] = 0.5f + row / 100f; + } + final QuantizedEmbeddingMatrix withWeights = + QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, 4, SEED) + .withPoolingWeights(weights); + final Path file = directory.resolve("weighted.bin"); + withWeights.write(file); + final QuantizedEmbeddingMatrix read = QuantizedEmbeddingMatrix.read(file); + assertArrayEquals(weights, read.poolingWeights(), 0f); + // Rewriting reproduces the file, so the weights block loses nothing. + final Path rewritten = directory.resolve("rewritten.bin"); + read.write(rewritten); + assertArrayEquals(Files.readAllBytes(file), Files.readAllBytes(rewritten)); + // Without weights the accessor answers null and the file omits the block. + final QuantizedEmbeddingMatrix withoutWeights = withWeights.withPoolingWeights(null); + assertEquals(null, withoutWeights.poolingWeights()); + assertTrue(Files.size(file) > sizeWithoutWeights(directory, withoutWeights), + "the weights block must add to the file size"); + } + + /** + * {@return the file size of a matrix written without weights} + * + * @param directory The directory to write into. + * @param matrix The matrix to write. + */ + private static long sizeWithoutWeights(Path directory, QuantizedEmbeddingMatrix matrix) + throws IOException { + final Path file = directory.resolve("unweighted.bin"); + matrix.write(file); + return Files.size(file); + } + + @Test + void testWithPoolingWeightsValidates() { + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(new float[4 * 8], 4, 8, 2, SEED); + assertThrows(IllegalArgumentException.class, + () -> quantized.withPoolingWeights(new float[3])); + assertThrows(IllegalArgumentException.class, + () -> quantized.withPoolingWeights(new float[] {1f, 2f, Float.NaN, 4f})); + } + @Test void testQuantizeValidatesItsArguments() { final float[] matrix = new float[2 * 4]; diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java new file mode 100644 index 0000000000..8ca0bf8d52 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The quantized model directory contract: after {@link ModelQuantizer} runs, the directory + * loads from {@code model.quantized} (with or without the safetensors still present), embeds + * and ranks like the float model up to the quantization error, and carries per-token pooling + * weights through the quantized file. + */ +class StaticEmbeddingModelQuantizedTest { + + private static final int DIMENSION = 32; + private static final String[] WORDS = { + "hello", "world", "apple", "banana", "cherry", "river", "mountain", "guitar", + "piano", "silver", "copper", "window" + }; + + /** + * Writes a small WordPiece model directory. + * + * @param directory The directory to write into. + * @param withWeights Whether to bundle a per-token {@code weights} tensor. + * @throws IOException Thrown if writing fails. + */ + private static void writeModelDirectory(Path directory, boolean withWeights) + throws IOException { + final List vocabulary = new ArrayList<>(List.of("[UNK]", "[CLS]", "[SEP]")); + vocabulary.addAll(List.of(WORDS)); + Files.write(directory.resolve("vocab.txt"), vocabulary); + final Random random = new Random(11); + final float[][] rows = new float[vocabulary.size()][DIMENSION]; + for (final float[] row : rows) { + final float rowScale = 0.5f + 2f * random.nextFloat(); + for (int d = 0; d < DIMENSION; d++) { + row[d] = rowScale * (float) random.nextGaussian(); + } + } + if (withWeights) { + final float[] weights = new float[vocabulary.size()]; + for (int row = 0; row < weights.length; row++) { + weights[row] = 0.5f + 1.5f * random.nextFloat(); + } + SafetensorsTestFiles.write(directory.resolve(ModelFileNames.SAFETENSORS), + SafetensorsTestFiles.matrix("embeddings", rows), + SafetensorsTestFiles.vector("weights", weights)); + } else { + SafetensorsTestFiles.write(directory.resolve(ModelFileNames.SAFETENSORS), + SafetensorsTestFiles.matrix("embeddings", rows)); + } + Files.writeString(directory.resolve(ModelFileNames.CONFIG), "{\"normalize\": true}"); + Files.writeString(directory.resolve(ModelFileNames.TOKENIZER_CONFIG), + "{\"do_lower_case\": true}"); + } + + @Test + void testQuantizedDirectoryEmbedsLikeTheFloatModel(@TempDir Path directory) + throws IOException { + writeModelDirectory(directory, false); + final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); + final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, 4, 7L); + assertEquals(WORDS.length + 3, result.rowCount()); + assertTrue(result.meanCosine() > 0.98, + "4-bit reconstruction reported mean cosine " + result.meanCosine()); + final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); + assertEquals(floatModel.dimension(), quantizedModel.dimension()); + for (final String text : new String[] {"hello world", "apple banana cherry", + "a guitar by the river", "no vocabulary hit here matches nothing"}) { + final double cosine = cosine(floatModel.embed(text), quantizedModel.embed(text)); + if (Double.isNaN(cosine)) { + // Both pooled to the zero vector: an out-of-vocabulary text, identical behavior. + continue; + } + assertTrue(cosine > 0.98, + "quantized embedding of '" + text + "' drifted to cosine " + cosine); + } + } + + @Test + void testQuantizedDirectoryLoadsWithoutTheSafetensors(@TempDir Path directory) + throws IOException { + writeModelDirectory(directory, false); + ModelQuantizer.quantize(directory, 4, 7L); + Files.delete(directory.resolve(ModelFileNames.SAFETENSORS)); + final StaticEmbeddingModel model = StaticEmbeddingModel.load(directory); + assertEquals(DIMENSION, model.dimension()); + assertEquals(WORDS.length + 3, model.vocabularySize()); + // The model's own row is its nearest neighbor, so ranking works end to end. + assertEquals("hello", model.mostSimilar("hello", 1).get(0).token()); + } + + @Test + void testPoolingWeightsRideThroughTheQuantizedFile(@TempDir Path directory) + throws IOException { + writeModelDirectory(directory, true); + final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); + final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, 4, 7L); + assertTrue(result.hasWeights(), "the weights tensor must be carried over"); + Files.delete(directory.resolve(ModelFileNames.SAFETENSORS)); + final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); + // Weighted pooling changes the pooled direction; matching the float model closely proves + // the weights arrived, since dropping them would score a visibly different vector. + final double cosine = + cosine(floatModel.embed("hello world apple"), quantizedModel.embed("hello world apple")); + assertTrue(cosine > 0.98, "weighted quantized embedding drifted to cosine " + cosine); + } + + @Test + void testMostSimilarAgreesWithTheFloatModel(@TempDir Path directory) throws IOException { + writeModelDirectory(directory, false); + final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); + ModelQuantizer.quantize(directory, 4, 7L); + final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); + for (final String word : new String[] {"hello", "river", "copper"}) { + assertEquals(floatModel.mostSimilar(word, 1).get(0).token(), + quantizedModel.mostSimilar(word, 1).get(0).token(), + "top neighbor of '" + word + "' must survive quantization"); + } + } + + @Test + void testQuantizedFileSmallerAndVerified(@TempDir Path directory) throws IOException { + writeModelDirectory(directory, false); + final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, 2, 7L); + assertTrue(result.quantizedBytes() < result.safetensorsBytes(), + result.quantizedBytes() + " must be smaller than " + result.safetensorsBytes()); + assertEquals(result.rowCount(), result.sampledRows(), + "a small table is verified row by row"); + assertTrue(result.meanCosine() > 0.9, + "2-bit reconstruction reported mean cosine " + result.meanCosine()); + } + + @Test + void testRowCountMismatchFailsLoud(@TempDir Path directory) throws IOException { + writeModelDirectory(directory, false); + ModelQuantizer.quantize(directory, 4, 7L); + final Path vocabularyFile = directory.resolve("vocab.txt"); + final List extended = new ArrayList<>(Files.readAllLines(vocabularyFile)); + extended.add("straggler"); + Files.write(vocabularyFile, extended); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(directory)); + assertTrue(e.getMessage().contains("do not belong to the same model"), e.getMessage()); + } + + @Test + void testQuantizerRequiresTheSafetensors(@TempDir Path directory) throws IOException { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> ModelQuantizer.quantize(directory, 4, 7L)); + assertTrue(e.getMessage().contains(ModelFileNames.SAFETENSORS), e.getMessage()); + } + + /** + * {@return the cosine between two vectors, or {@code Double.NaN} when either has no + * direction} + * + * @param a The first vector. + * @param b The second vector, of the same length. + */ + private static double cosine(float[] a, float[] b) { + double dot = 0; + double normASquared = 0; + double normBSquared = 0; + for (int d = 0; d < a.length; d++) { + dot += (double) a[d] * b[d]; + normASquared += (double) a[d] * a[d]; + normBSquared += (double) b[d] * b[d]; + } + final double denominator = Math.sqrt(normASquared) * Math.sqrt(normBSquared); + return denominator == 0 ? Double.NaN : dot / denominator; + } +} From add1add236fc16adf716c04c144942875b6f2a91 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 24 Jul 2026 00:17:23 -0400 Subject: [PATCH 73/82] OPENNLP-1895: Reject ambiguous matrix source, widen the seed, document quantization, expand tests Fail loud when a directory holds both model.quantized and model.safetensors instead of preferring one silently. Widen the QuantizeModel seed argument to long so the full seed space is expressible. Add a Quantized Models manual section pointing at the workflow tests. Convert reconstruction and embed-parity tests to parameterized bit-width cases, add the SentencePiece quantized path and a both-files-present rejection, and share the cosine and SentencePiece fixture helpers. --- opennlp-docs/src/docbkx/embeddings.xml | 39 +++++ .../opennlp/embeddings/EmbeddingTable.java | 20 ++- .../embeddings/FloatEmbeddingTable.java | 8 + .../opennlp/embeddings/GaussianQuantizer.java | 10 +- .../opennlp/embeddings/ModelFileNames.java | 5 +- .../opennlp/embeddings/ModelQuantizer.java | 11 +- .../embeddings/QuantizedEmbeddingMatrix.java | 15 +- .../embeddings/QuantizedTableAdapter.java | 8 + .../cmdline/QuantizeModelParams.java | 2 +- .../embeddings/HadamardRotationTest.java | 11 -- .../QuantizedEmbeddingMatrixTest.java | 50 ++---- .../embeddings/SentencePieceModelFixture.java | 151 ++++++++++++++++++ .../StaticEmbeddingModelQuantizedTest.java | 147 ++++++++++++----- ...eddingModelSentencePieceQuantizedTest.java | 102 ++++++++++++ 14 files changed, 457 insertions(+), 122 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SentencePieceModelFixture.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 69fd85d1b9..488739608d 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -224,6 +224,45 @@ results.sort(Comparator.comparingDouble(Scored::score).reversed());]]>
+
+ Quantized Models + + A static embedding table can be quantized to 2, 3, or 4 bits per dimension, shrinking + it by roughly 8 to 16 times against the 32-bit float matrix (a 500,000-row, + 300-dimension table drops from about 600 MB to 77 MB at 4 bits). Because embedding is + memory-bound row gathering, reading fewer bytes is also the throughput lever. The + method is the TurboQuant construction: each row is rotated so its coordinates become + near-independent and near-Gaussian, and each rotated coordinate is encoded against an + optimal scalar grid, with a least-squares scale fitted per row. + + + The QuantizeModel tool quantizes a model directory's + model.safetensors in place, writing model.quantized next to + it and reporting the sizes and the reconstruction quality it measured from the written + file: + + + + + + A directory presents exactly one matrix file. After quantizing, delete the + model.safetensors to deploy the quantized matrix; the loader then reads + model.quantized with no code change, and any per-token pooling weights + are carried inside the quantized file. A directory that still holds both files is + rejected at load time, so the deployment must choose which matrix is authoritative. + Embedding, similarity, and mostSimilar behave the same as on + the float model, up to the quantization error of the chosen bit width; fewer bits + trade fidelity for size. + + + StaticEmbeddingModelQuantizedTest asserts this workflow end to end: + quantizing a directory, loading it after the safetensors is removed, and matching the + float model's embeddings and rankings; QuantizedEmbeddingMatrixTest pins + the reconstruction quality at each bit width. + +
+
The safetensors Reader diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java index a4c89ec5b0..efb4efacf8 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/EmbeddingTable.java @@ -18,19 +18,17 @@ /** * The row storage behind {@link StaticEmbeddingModel}: gathering rows into a pooled vector and - * scoring rows against a query, independent of whether the rows are float or quantized. + * scoring rows against a query. * - *

The seam is shaped so a storage form may pool in a working space of its own. - * {@link #addRow(int, float, float[])} accumulates into a vector of {@link #pooledLength()}, and - * {@link #finishPooling(float[])} maps the accumulated vector to original space once per pooled - * result. The float table's working space is original space and its finish is the identity; the - * quantized table pools in rotated space, where decoding a row is a grid lookup, and pays its - * single inverse rotation in the finish. Scoring mirrors this: {@link #prepareQuery(float[])} - * maps a query into the working space once, and {@link #dot(int, float[])} scores every row - * against the prepared query there.

+ *

A table may work in a space of its own choosing. {@link #addRow(int, float, float[])} + * accumulates into a vector of {@link #pooledLength()}, and {@link #finishPooling(float[])} + * maps the accumulated vector to original space once per pooled result. Scoring mirrors this: + * {@link #prepareQuery(float[])} maps a query into the working space once, and + * {@link #dot(int, float[])} scores every row against the prepared query there. The working + * space must preserve norms and dot products, so cosine math is space-independent.

* - *

Implementations are immutable and safe for concurrent use; the accumulator and prepared - * query arrays belong to the caller.

+ *

Implementations must be safe for concurrent use; the accumulator and prepared query + * arrays belong to the caller.

*/ interface EmbeddingTable { diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java index 2230862707..7b1f7f7658 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/FloatEmbeddingTable.java @@ -53,21 +53,25 @@ final class FloatEmbeddingTable implements EmbeddingTable { } } + /** {@inheritDoc} */ @Override public int rowCount() { return rowCount; } + /** {@inheritDoc} */ @Override public int dimension() { return dimension; } + /** {@inheritDoc} */ @Override public int pooledLength() { return dimension; } + /** {@inheritDoc} */ @Override public void addRow(int row, float weight, float[] sum) { final int base = row * dimension; @@ -82,16 +86,19 @@ public void addRow(int row, float weight, float[] sum) { } } + /** {@inheritDoc} */ @Override public float[] finishPooling(float[] sum) { return sum; } + /** {@inheritDoc} */ @Override public float[] prepareQuery(float[] query) { return query; } + /** {@inheritDoc} */ @Override public double dot(int row, float[] preparedQuery) { final int base = row * dimension; @@ -114,6 +121,7 @@ public double dot(int row, float[] preparedQuery) { return dot; } + /** {@inheritDoc} */ @Override public double rowNorm(int row) { return rowNorms[row]; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java index 898ac77433..1339d1376d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/GaussianQuantizer.java @@ -23,11 +23,13 @@ * minimizing the mean squared error over {@code N(0,1)}, with encoding by nearest level. The * coordinates of a {@link HadamardRotation rotated} unit vector, scaled by the square root of the * padded dimension, follow this distribution closely, which is what makes one fixed grid - * near-optimal for every coordinate of every vector (Zandieh et al., TurboQuant: Online Vector - * Quantization with Near-optimal Distortion Rate, arXiv:2504.19874). + * near-optimal for every coordinate of every vector (Zandieh et al., + * TurboQuant: Online Vector Quantization with + * Near-optimal Distortion Rate). * - *

The levels are the classic Lloyd-Max quantizer of the Gaussian (Max, Quantizing for - * minimum distortion, IRE Transactions on Information Theory, 1960), computed here by Lloyd + *

The levels are the classic Lloyd-Max quantizer of the Gaussian (Max, + * Quantizing for minimum + * distortion, IRE Transactions on Information Theory, 1960), computed here by Lloyd * iteration over a fine discretization of the density rather than copied from published tables, * so the derivation is in this file and reproducible. Computed grids are cached per bit width. * Encoding compares against the midpoints between adjacent levels, which is exactly the diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java index 2a48d2c9b1..b7ee8874cd 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java @@ -37,8 +37,9 @@ final class ModelFileNames { static final String SAFETENSORS = "model.safetensors"; /** - * The quantized matrix file, written by the {@code QuantizeModel} tool. When present it wins - * over {@link #SAFETENSORS}: it carries the matrix and any per-token weights itself. + * The quantized matrix file, written by the {@code QuantizeModel} tool. It carries the matrix + * and any per-token weights itself, and is the directory's matrix source in place of + * {@link #SAFETENSORS}, which a quantized deployment deletes. */ static final String QUANTIZED = "model.quantized"; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java index c440bf1b13..7fd04d61ef 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java @@ -93,10 +93,11 @@ public static Result quantize(Path modelDirectory, int bits, long seed) throws I final int dimension = matrixInfo.shape()[1]; final float[] matrix = tensors.readFloats(matrixName); float[] weights = null; - if (tensors.tensorNames().contains("weights")) { - weights = tensors.readFloats("weights"); + if (tensors.tensorNames().contains(StaticEmbeddingModel.WEIGHTS_TENSOR_NAME)) { + weights = tensors.readFloats(StaticEmbeddingModel.WEIGHTS_TENSOR_NAME); if (weights.length != rowCount) { - throw new IllegalArgumentException("Tensor 'weights' in " + safetensorsFile + " has " + throw new IllegalArgumentException("Tensor '" + + StaticEmbeddingModel.WEIGHTS_TENSOR_NAME + "' in " + safetensorsFile + " has " + weights.length + " elements but the matrix has " + rowCount + " rows"); } } @@ -125,14 +126,14 @@ public static Result quantize(Path modelDirectory, int bits, long seed) throws I /** * {@return the cosine between a matrix row and its reconstruction, or {@code Double.NaN} when - * either has no direction} + * either has no direction} Also the shared fidelity measure of this package's tests. * * @param matrix The flat row-major matrix. * @param base The row's first index. * @param dimension The row width. * @param decoded The reconstructed row. */ - private static double cosine(float[] matrix, int base, int dimension, float[] decoded) { + static double cosine(float[] matrix, int base, int dimension, float[] decoded) { double dot = 0; double normASquared = 0; double normBSquared = 0; diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java index cbddaaa56a..4adfc51940 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java @@ -32,8 +32,9 @@ /** * An embedding matrix quantized to {@code 2}-{@code 4} bits per dimension, following the - * TurboQuant construction (Zandieh, Daliri, Hadian, Mirrokni, TurboQuant: Online Vector - * Quantization with Near-optimal Distortion Rate, arXiv:2504.19874): each row is rotated by + * TurboQuant construction (Zandieh, Daliri, Hadian, Mirrokni, + * TurboQuant: Online Vector Quantization with + * Near-optimal Distortion Rate): each row is rotated by * a seeded {@link HadamardRotation}, so its coordinates become near-independent and * near-Gaussian, and each rotated coordinate is encoded independently with the * {@link GaussianQuantizer} grid of the chosen bit width. A row decodes to a per-row scale times @@ -612,14 +613,4 @@ private static void writeCode(byte[] codes, int rowBase, int bits, int index, in codes[byteIndex + 1] |= (byte) (code >>> (8 - shift)); } } - - /** - * {@return this row's code at an index, for tests} - * - * @param row The row. - * @param index The code index within the row, up to the padded dimension. - */ - int code(int row, int index) { - return readCode(codes, row * rowBytes, bits, index); - } } diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java index 868ec39878..62dc180f78 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedTableAdapter.java @@ -35,41 +35,49 @@ final class QuantizedTableAdapter implements EmbeddingTable { this.matrix = matrix; } + /** {@inheritDoc} */ @Override public int rowCount() { return matrix.rowCount(); } + /** {@inheritDoc} */ @Override public int dimension() { return matrix.dimension(); } + /** {@inheritDoc} */ @Override public int pooledLength() { return matrix.paddedDimension(); } + /** {@inheritDoc} */ @Override public void addRow(int row, float weight, float[] sum) { matrix.addRowRotated(row, weight, sum); } + /** {@inheritDoc} */ @Override public float[] finishPooling(float[] sum) { return matrix.toOriginal(sum); } + /** {@inheritDoc} */ @Override public float[] prepareQuery(float[] query) { return matrix.rotate(query); } + /** {@inheritDoc} */ @Override public double dot(int row, float[] preparedQuery) { return matrix.dotRotated(row, preparedQuery); } + /** {@inheritDoc} */ @Override public double rowNorm(int row) { return matrix.rowNorm(row); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java index ab79d9ba1c..151035417f 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/QuantizeModelParams.java @@ -47,5 +47,5 @@ interface QuantizeModelParams { @ParameterDescription(valueName = "seed", description = "the rotation seed; the same matrix, bits, and seed write the same file") @OptionalParameter(defaultValue = "0") - Integer getSeed(); + Long getSeed(); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java index d8c38de364..013f31086b 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/HadamardRotationTest.java @@ -25,7 +25,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * The rotation contract: orthonormal (norms and dot products preserved), self-consistent @@ -142,14 +141,4 @@ private static double dot(float[] a, float[] b) { } return dot; } - - @Test - void testHelpersKeepThePaddedContract() { - // The helpers above assume rotate() leaves length unchanged; pin that here. - final HadamardRotation rotation = new HadamardRotation(3, 0L); - final float[] vector = new float[] {1f, 2f, 3f, 0f}; - rotation.rotate(vector); - assertEquals(4, vector.length); - assertTrue(Float.isFinite(vector[0])); - } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java index f7304299ca..53d5accb0c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java @@ -24,9 +24,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -57,31 +60,20 @@ private static float[] testMatrix() { return matrix; } - @Test - void testReconstructionQualityPerBitWidth() { - // The mean squared error of the Gaussian Lloyd-Max grids translates to an expected cosine - // between a row and its reconstruction; these thresholds sit safely below the analytic - // expectation (about 0.945 at 2 bits, 0.983 at 3, 0.995 at 4) but far above what a broken - // rotation, grid, or scale would produce. - assertMeanCosineAtLeast(2, 0.92); - assertMeanCosineAtLeast(3, 0.97); - assertMeanCosineAtLeast(4, 0.99); - } - - /** - * Asserts the mean cosine between original and decoded rows for a bit width. - * - * @param bits The bit width under test. - * @param threshold The minimum acceptable mean cosine. - */ - private static void assertMeanCosineAtLeast(int bits, double threshold) { + // The mean squared error of the Gaussian Lloyd-Max grids translates to an expected cosine + // between a row and its reconstruction; these thresholds sit safely below the analytic + // expectation (about 0.945 at 2 bits, 0.983 at 3, 0.995 at 4) but far above what a broken + // rotation, grid, or scale would produce. + @ParameterizedTest + @CsvSource({"2, 0.92", "3, 0.97", "4, 0.99"}) + void testReconstructionQualityPerBitWidth(int bits, double threshold) { final float[] matrix = testMatrix(); final QuantizedEmbeddingMatrix quantized = QuantizedEmbeddingMatrix.quantize(matrix, ROWS, DIMENSION, bits, SEED); double cosineSum = 0; for (int row = 0; row < ROWS; row++) { final float[] decoded = quantized.decodeRow(row); - cosineSum += cosine(matrix, row * DIMENSION, decoded); + cosineSum += ModelQuantizer.cosine(matrix, row * DIMENSION, DIMENSION, decoded); } final double meanCosine = cosineSum / ROWS; assertTrue(meanCosine >= threshold, bits + " bits reconstructed a mean cosine of " @@ -227,7 +219,7 @@ void testPoolingWeightsRoundTripThroughTheFile(@TempDir Path directory) throws I assertArrayEquals(Files.readAllBytes(file), Files.readAllBytes(rewritten)); // Without weights the accessor answers null and the file omits the block. final QuantizedEmbeddingMatrix withoutWeights = withWeights.withPoolingWeights(null); - assertEquals(null, withoutWeights.poolingWeights()); + assertNull(withoutWeights.poolingWeights()); assertTrue(Files.size(file) > sizeWithoutWeights(directory, withoutWeights), "the weights block must add to the file size"); } @@ -316,22 +308,4 @@ void testReadRejectsForeignAndTruncatedFiles(@TempDir Path directory) throws IOE assertThrows(IllegalArgumentException.class, () -> QuantizedEmbeddingMatrix.read(trailing)); } - /** - * {@return the cosine between a matrix row and a decoded vector} - * - * @param matrix The flat row-major matrix. - * @param base The row's first index. - * @param decoded The decoded row. - */ - private static double cosine(float[] matrix, int base, float[] decoded) { - double dot = 0; - double normASquared = 0; - double normBSquared = 0; - for (int d = 0; d < decoded.length; d++) { - dot += (double) matrix[base + d] * decoded[d]; - normASquared += (double) matrix[base + d] * matrix[base + d]; - normBSquared += (double) decoded[d] * decoded[d]; - } - return dot / (Math.sqrt(normASquared) * Math.sqrt(normBSquared)); - } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SentencePieceModelFixture.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SentencePieceModelFixture.java new file mode 100644 index 0000000000..7c9571c980 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/SentencePieceModelFixture.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import opennlp.subword.sentencepiece.SentencePieceTokenizer; + +/** + * Writes a SentencePiece-layout static embedding model directory around the bundled tiny + * Unigram test model, for tests that need one. The matrix vocabulary is written the way a + * distillation ships it: control pieces dropped, extra special rows in front, and one token + * appended through {@code added_tokens}, so lookups must go by piece string, not tokenizer id. + */ +final class SentencePieceModelFixture { + + static final String MODEL_RESOURCE = "/opennlp/embeddings/tiny-unigram.model"; + + private final byte[] modelBytes; + private final SentencePieceTokenizer tokenizer; + // The matrix rows: , , then every poolable tokenizer piece. + private final List rows; + + /** + * Loads the bundled model resource and derives the row order. + * + * @throws IOException Thrown if the resource cannot be read. + */ + SentencePieceModelFixture() throws IOException { + try (InputStream in = SentencePieceModelFixture.class.getResourceAsStream(MODEL_RESOURCE)) { + modelBytes = in.readAllBytes(); + } + tokenizer = SentencePieceTokenizer.load( + SentencePieceModelFixture.class.getResourceAsStream(MODEL_RESOURCE)); + rows = new ArrayList<>(); + rows.add(""); + rows.add(""); + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + if (!tokenizer.isControl(id) && !tokenizer.isUnknown(id)) { + rows.add(tokenizer.idToPiece(id)); + } + } + } + + /** {@return the loaded tokenizer} */ + SentencePieceTokenizer tokenizer() { + return tokenizer; + } + + /** {@return the matrix row pieces, in row order, with the appended added token last} */ + List rowPieces() { + final List pieces = new ArrayList<>(rows); + pieces.add(""); + return pieces; + } + + /** + * Writes the {@code .model}, a synthesized Unigram {@code tokenizer.json}, a deterministic + * embedding matrix, and a {@code config.json} into a directory. + * + * @param directory The directory to write into. + * @param dimension The embedding dimension. + * @param normalize The {@code config.json} normalize value. + * @param seed The seed of the deterministic matrix values. + * @throws IOException Thrown if writing fails. + */ + void write(Path directory, int dimension, boolean normalize, long seed) throws IOException { + Files.write(directory.resolve("sentencepiece.bpe.model"), modelBytes); + Files.writeString(directory.resolve("tokenizer.json"), tokenizerJson(rows)); + final int rowCount = rows.size() + 1; + final float[][] matrix = new float[rowCount][dimension]; + final Random random = new Random(seed); + for (final float[] row : matrix) { + final float rowScale = 0.5f + 2f * random.nextFloat(); + for (int d = 0; d < dimension; d++) { + row[d] = rowScale * (float) random.nextGaussian(); + } + } + SafetensorsTestFiles.write(directory.resolve(ModelFileNames.SAFETENSORS), + SafetensorsTestFiles.matrix("embeddings", matrix)); + Files.writeString(directory.resolve(ModelFileNames.CONFIG), + "{\"model_type\":\"model2vec\",\"normalize\":" + normalize + "}"); + } + + /** + * {@return a Unigram {@code tokenizer.json} whose vocabulary is the given pieces plus an + * appended added token} The appended token is not {@code } because the fixture model + * defines {@code } as a user-defined piece that already owns a row. + * + * @param pieces The {@code model.vocab} pieces in row order. + */ + private static String tokenizerJson(List pieces) { + final StringBuilder json = new StringBuilder("{\"version\":\"1.0\",\"added_tokens\":["); + json.append("{\"id\":0,\"content\":\"\",\"special\":true},"); + json.append("{\"id\":").append(pieces.size()).append(",\"content\":\"\"," + + "\"special\":true}],"); + json.append("\"normalizer\":{\"type\":\"Precompiled\"},\"model\":{\"type\":\"Unigram\"," + + "\"unk_id\":1,\"vocab\":["); + for (int i = 0; i < pieces.size(); i++) { + if (i > 0) { + json.append(','); + } + json.append('[').append(quote(pieces.get(i))).append(",-").append(i % 7).append(".5]"); + } + return json.append("]}}").toString(); + } + + /** + * {@return {@code text} as a JSON string literal} + * + * @param text The text to quote. + */ + private static String quote(String text) { + final StringBuilder quoted = new StringBuilder("\""); + for (int i = 0; i < text.length(); i++) { + final char c = text.charAt(i); + switch (c) { + case '"' -> quoted.append("\\\""); + case '\\' -> quoted.append("\\\\"); + default -> { + if (c < 0x20) { + quoted.append(String.format("\\u%04x", (int) c)); + } else { + quoted.append(c); + } + } + } + } + return quoted.append('"').toString(); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java index 8ca0bf8d52..0e41df7bf6 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java @@ -25,24 +25,34 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * The quantized model directory contract: after {@link ModelQuantizer} runs, the directory - * loads from {@code model.quantized} (with or without the safetensors still present), embeds - * and ranks like the float model up to the quantization error, and carries per-token pooling - * weights through the quantized file. + * The quantized WordPiece model directory contract: after {@link ModelQuantizer} runs and the + * safetensors is removed, the directory loads from {@code model.quantized}, embeds and ranks + * like the float model up to the quantization error, and carries per-token pooling weights + * through the quantized file. A directory holding both matrix files is rejected. */ class StaticEmbeddingModelQuantizedTest { private static final int DIMENSION = 32; + private static final long SEED = 7L; private static final String[] WORDS = { "hello", "world", "apple", "banana", "cherry", "river", "mountain", "guitar", "piano", "silver", "copper", "window" }; + private static final String[] SENTENCES = { + "hello world", "apple banana cherry", "a guitar by the river", + "no vocabulary hit here matches nothing" + }; /** * Writes a small WordPiece model directory. @@ -71,7 +81,7 @@ private static void writeModelDirectory(Path directory, boolean withWeights) } SafetensorsTestFiles.write(directory.resolve(ModelFileNames.SAFETENSORS), SafetensorsTestFiles.matrix("embeddings", rows), - SafetensorsTestFiles.vector("weights", weights)); + SafetensorsTestFiles.vector(StaticEmbeddingModel.WEIGHTS_TENSOR_NAME, weights)); } else { SafetensorsTestFiles.write(directory.resolve(ModelFileNames.SAFETENSORS), SafetensorsTestFiles.matrix("embeddings", rows)); @@ -81,35 +91,55 @@ private static void writeModelDirectory(Path directory, boolean withWeights) "{\"do_lower_case\": true}"); } - @Test - void testQuantizedDirectoryEmbedsLikeTheFloatModel(@TempDir Path directory) + /** + * Quantizes the directory and removes the safetensors, leaving the quantized deployment the + * loader accepts. + * + * @param directory The model directory to quantize in place. + * @param bits The bit width. + * @return What the quantizer measured. + * @throws IOException Thrown if quantizing or deleting fails. + */ + private static ModelQuantizer.Result deployQuantized(Path directory, int bits) + throws IOException { + final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, bits, SEED); + Files.delete(directory.resolve(ModelFileNames.SAFETENSORS)); + return result; + } + + @ParameterizedTest + @ValueSource(ints = {2, 3, 4}) + void testQuantizedDirectoryEmbedsLikeTheFloatModel(int bits, @TempDir Path directory) throws IOException { writeModelDirectory(directory, false); final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); - final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, 4, 7L); - assertEquals(WORDS.length + 3, result.rowCount()); - assertTrue(result.meanCosine() > 0.98, - "4-bit reconstruction reported mean cosine " + result.meanCosine()); + deployQuantized(directory, bits); final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); assertEquals(floatModel.dimension(), quantizedModel.dimension()); - for (final String text : new String[] {"hello world", "apple banana cherry", - "a guitar by the river", "no vocabulary hit here matches nothing"}) { + // Pooling several rows accumulates independent quantization noise, so the pooled cosine + // sits a little below the single-row reconstruction; the floor loosens as bits shrink. + // Every value is far above what a broken rotation, grid, or scale would produce. + final double threshold = switch (bits) { + case 2 -> 0.88; + case 3 -> 0.95; + default -> 0.98; + }; + for (final String text : SENTENCES) { final double cosine = cosine(floatModel.embed(text), quantizedModel.embed(text)); if (Double.isNaN(cosine)) { // Both pooled to the zero vector: an out-of-vocabulary text, identical behavior. continue; } - assertTrue(cosine > 0.98, - "quantized embedding of '" + text + "' drifted to cosine " + cosine); + assertTrue(cosine >= threshold, + bits + "-bit embedding of '" + text + "' drifted to cosine " + cosine); } } @Test - void testQuantizedDirectoryLoadsWithoutTheSafetensors(@TempDir Path directory) + void testQuantizedDirectoryLoadsAfterTheSafetensorsIsRemoved(@TempDir Path directory) throws IOException { writeModelDirectory(directory, false); - ModelQuantizer.quantize(directory, 4, 7L); - Files.delete(directory.resolve(ModelFileNames.SAFETENSORS)); + deployQuantized(directory, 4); final StaticEmbeddingModel model = StaticEmbeddingModel.load(directory); assertEquals(DIMENSION, model.dimension()); assertEquals(WORDS.length + 3, model.vocabularySize()); @@ -117,14 +147,27 @@ void testQuantizedDirectoryLoadsWithoutTheSafetensors(@TempDir Path directory) assertEquals("hello", model.mostSimilar("hello", 1).get(0).token()); } + @Test + void testBothMatrixFilesPresentIsRejected(@TempDir Path directory) throws IOException { + writeModelDirectory(directory, false); + // ModelQuantizer writes model.quantized next to model.safetensors and leaves both. + ModelQuantizer.quantize(directory, 4, SEED); + assertTrue(Files.isRegularFile(directory.resolve(ModelFileNames.SAFETENSORS))); + assertTrue(Files.isRegularFile(directory.resolve(ModelFileNames.QUANTIZED))); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(directory)); + assertTrue(e.getMessage().contains("has both"), e.getMessage()); + assertTrue(e.getMessage().contains(ModelFileNames.QUANTIZED), e.getMessage()); + assertTrue(e.getMessage().contains(ModelFileNames.SAFETENSORS), e.getMessage()); + } + @Test void testPoolingWeightsRideThroughTheQuantizedFile(@TempDir Path directory) throws IOException { writeModelDirectory(directory, true); final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); - final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, 4, 7L); + final ModelQuantizer.Result result = deployQuantized(directory, 4); assertTrue(result.hasWeights(), "the weights tensor must be carried over"); - Files.delete(directory.resolve(ModelFileNames.SAFETENSORS)); final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); // Weighted pooling changes the pooled direction; matching the float model closely proves // the weights arrived, since dropping them would score a visibly different vector. @@ -137,7 +180,7 @@ void testPoolingWeightsRideThroughTheQuantizedFile(@TempDir Path directory) void testMostSimilarAgreesWithTheFloatModel(@TempDir Path directory) throws IOException { writeModelDirectory(directory, false); final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); - ModelQuantizer.quantize(directory, 4, 7L); + deployQuantized(directory, 4); final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); for (final String word : new String[] {"hello", "river", "copper"}) { assertEquals(floatModel.mostSimilar(word, 1).get(0).token(), @@ -147,21 +190,36 @@ void testMostSimilarAgreesWithTheFloatModel(@TempDir Path directory) throws IOEx } @Test - void testQuantizedFileSmallerAndVerified(@TempDir Path directory) throws IOException { + void testAnalogyRunsOverTheQuantizedTable(@TempDir Path directory) throws IOException { + writeModelDirectory(directory, false); + deployQuantized(directory, 4); + final StaticEmbeddingModel model = StaticEmbeddingModel.load(directory); + // The three query terms are excluded, so a fourth vocabulary word comes back; the point is + // that the analogy path (query build, exclusion, scan) runs end to end over rotated space. + final List neighbors = model.analogy("hello", "world", "apple", 1); + assertEquals(1, neighbors.size()); + // The analogy excludes its own terms, so none of them comes back. + assertFalse(neighbors.get(0).token().equals("apple")); + } + + @ParameterizedTest + @CsvSource({"2, 0.9", "4, 0.98"}) + void testQuantizedFileSmallerAndVerified(int bits, double minCosine, @TempDir Path directory) + throws IOException { writeModelDirectory(directory, false); - final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, 2, 7L); + final ModelQuantizer.Result result = ModelQuantizer.quantize(directory, bits, SEED); assertTrue(result.quantizedBytes() < result.safetensorsBytes(), result.quantizedBytes() + " must be smaller than " + result.safetensorsBytes()); assertEquals(result.rowCount(), result.sampledRows(), "a small table is verified row by row"); - assertTrue(result.meanCosine() > 0.9, - "2-bit reconstruction reported mean cosine " + result.meanCosine()); + assertTrue(result.meanCosine() > minCosine, + bits + "-bit reconstruction reported mean cosine " + result.meanCosine()); } @Test void testRowCountMismatchFailsLoud(@TempDir Path directory) throws IOException { writeModelDirectory(directory, false); - ModelQuantizer.quantize(directory, 4, 7L); + deployQuantized(directory, 4); final Path vocabularyFile = directory.resolve("vocab.txt"); final List extended = new ArrayList<>(Files.readAllLines(vocabularyFile)); extended.add("straggler"); @@ -172,12 +230,34 @@ void testRowCountMismatchFailsLoud(@TempDir Path directory) throws IOException { } @Test - void testQuantizerRequiresTheSafetensors(@TempDir Path directory) throws IOException { + void testQuantizerRequiresTheSafetensors(@TempDir Path directory) { final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> ModelQuantizer.quantize(directory, 4, 7L)); + () -> ModelQuantizer.quantize(directory, 4, SEED)); assertTrue(e.getMessage().contains(ModelFileNames.SAFETENSORS), e.getMessage()); } + @Test + void testQuantizerRejectsBadBitWidths(@TempDir Path directory) throws IOException { + writeModelDirectory(directory, false); + assertThrows(IllegalArgumentException.class, () -> ModelQuantizer.quantize(directory, 1, SEED)); + assertThrows(IllegalArgumentException.class, () -> ModelQuantizer.quantize(directory, 5, SEED)); + } + + @Test + void testQuantizedEmbeddingsAreDeterministic(@TempDir Path first, @TempDir Path second) + throws IOException { + writeModelDirectory(first, false); + writeModelDirectory(second, false); + deployQuantized(first, 4); + deployQuantized(second, 4); + final StaticEmbeddingModel modelA = StaticEmbeddingModel.load(first); + final StaticEmbeddingModel modelB = StaticEmbeddingModel.load(second); + for (final String text : SENTENCES) { + assertArrayEquals(modelA.embed(text), modelB.embed(text), 0f, + "the same table, bits, and seed must embed bit-identically"); + } + } + /** * {@return the cosine between two vectors, or {@code Double.NaN} when either has no * direction} @@ -186,15 +266,6 @@ void testQuantizerRequiresTheSafetensors(@TempDir Path directory) throws IOExcep * @param b The second vector, of the same length. */ private static double cosine(float[] a, float[] b) { - double dot = 0; - double normASquared = 0; - double normBSquared = 0; - for (int d = 0; d < a.length; d++) { - dot += (double) a[d] * b[d]; - normASquared += (double) a[d] * a[d]; - normBSquared += (double) b[d] * b[d]; - } - final double denominator = Math.sqrt(normASquared) * Math.sqrt(normBSquared); - return denominator == 0 ? Double.NaN : dot / denominator; + return ModelQuantizer.cosine(a, 0, a.length, b); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java new file mode 100644 index 0000000000..4f43066bda --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The quantized SentencePiece loading path: a SentencePiece directory whose matrix has been + * quantized and whose safetensors removed loads through {@code loadSentencePieceQuantized}, + * resolves rows by piece string across the tokenizer id offset, and embeds like the float + * model up to the quantization error. Both matrix files present is rejected. + */ +class StaticEmbeddingModelSentencePieceQuantizedTest { + + private static final int DIMENSION = 16; + private static final long SEED = 7L; + + private static SentencePieceModelFixture fixture; + + @BeforeAll + static void loadFixture() throws IOException { + fixture = new SentencePieceModelFixture(); + } + + @Test + void testQuantizedSentencePieceEmbedsLikeTheFloatModel(@TempDir Path directory) + throws IOException { + fixture.write(directory, DIMENSION, true, SEED); + final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); + ModelQuantizer.quantize(directory, 4, SEED); + Files.delete(directory.resolve(ModelFileNames.SAFETENSORS)); + final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); + assertEquals(floatModel.dimension(), quantizedModel.dimension()); + assertEquals(floatModel.vocabularySize(), quantizedModel.vocabularySize()); + for (final String text : new String[] {"a", "the model", "hello there world"}) { + final double cosine = cosine(floatModel.embed(text), quantizedModel.embed(text)); + if (Double.isNaN(cosine)) { + continue; + } + assertTrue(cosine > 0.97, + "quantized SentencePiece embedding of '" + text + "' drifted to cosine " + cosine); + } + } + + @Test + void testQuantizedSentencePieceRanksLikeTheFloatModel(@TempDir Path directory) + throws IOException { + fixture.write(directory, DIMENSION, true, SEED); + final StaticEmbeddingModel floatModel = StaticEmbeddingModel.load(directory); + ModelQuantizer.quantize(directory, 4, SEED); + Files.delete(directory.resolve(ModelFileNames.SAFETENSORS)); + final StaticEmbeddingModel quantizedModel = StaticEmbeddingModel.load(directory); + // A piece is its own nearest neighbor under both storage forms. + final String piece = fixture.rowPieces().get(3); + assertEquals(floatModel.mostSimilar(piece, 1).get(0).token(), + quantizedModel.mostSimilar(piece, 1).get(0).token()); + } + + @Test + void testBothMatrixFilesPresentIsRejected(@TempDir Path directory) throws IOException { + fixture.write(directory, DIMENSION, true, SEED); + ModelQuantizer.quantize(directory, 4, SEED); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(directory)); + assertTrue(e.getMessage().contains("has both"), e.getMessage()); + } + + /** + * {@return the cosine between two vectors, or {@code Double.NaN} when either has no + * direction} + * + * @param a The first vector. + * @param b The second vector, of the same length. + */ + private static double cosine(float[] a, float[] b) { + return ModelQuantizer.cosine(a, 0, a.length, b); + } +} From b6bee0ac7244cb48e3d9019f56925b3f964d5a95 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 24 Jul 2026 00:34:38 -0400 Subject: [PATCH 74/82] OPENNLP-1895: Reject dimensions whose padded bit count overflows an int A quantized file header declaring a dimension near 2^29 made paddedDimension * bits overflow a signed int, so the per-row byte count went negative and reading the file crashed with an undocumented NegativeArraySizeException instead of a clean rejection; the same overflow would corrupt the bit addressing in readCode/writeCode. Compute the row byte count in long arithmetic through a single range-checked helper used by quantize, read, and the constructor. Add edge-case tests across non-power-of-two dimensions, a one-dimensional matrix, and adversarial rows the rotation must still reconstruct. --- .../embeddings/QuantizedEmbeddingMatrix.java | 26 +++- .../QuantizedEmbeddingMatrixEdgeCaseTest.java | 133 ++++++++++++++++++ .../QuantizedEmbeddingMatrixTest.java | 20 +++ 3 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixEdgeCaseTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java index 4adfc51940..7a8567d663 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java @@ -108,7 +108,7 @@ private QuantizedEmbeddingMatrix(int rowCount, int dimension, int bits, long see this.paddedDimension = HadamardRotation.paddedDimension(dimension); this.bits = bits; this.seed = seed; - this.rowBytes = (paddedDimension * bits + 7) / 8; + this.rowBytes = rowByteCount(paddedDimension, bits); this.quantizer = quantizer; this.rotation = new HadamardRotation(dimension, seed); this.scales = scales; @@ -152,7 +152,7 @@ public static QuantizedEmbeddingMatrix quantize(float[] rowMajor, int rowCount, final GaussianQuantizer quantizer = GaussianQuantizer.forBits(bits); final HadamardRotation rotation = new HadamardRotation(dimension, seed); final int paddedDimension = rotation.paddedDimension(); - final int rowBytes = (paddedDimension * bits + 7) / 8; + final int rowBytes = rowByteCount(paddedDimension, bits); requireStorableSize(rowCount, rowBytes); final float[] scales = new float[rowCount]; final byte[] codes = new byte[rowCount * rowBytes]; @@ -269,6 +269,26 @@ private static void requireStorableSize(int rowCount, int rowBytes) { } } + /** + * {@return the packed byte count of one row} Computed in long arithmetic and range-checked, so + * a padded dimension large enough to overflow {@code paddedDimension * bits} as a signed int + * (which would silently produce a negative or wrapped byte count) is rejected instead. + * + * @param paddedDimension The power-of-two padded dimension. + * @param bits The bit width per padded dimension. + * @throws IllegalArgumentException Thrown if the padded bit count exceeds what an {@code int} + * can address. + */ + private static int rowByteCount(int paddedDimension, int bits) { + final long paddedBits = (long) paddedDimension * bits; + if (paddedBits > Integer.MAX_VALUE - 7) { + throw new IllegalArgumentException("A padded dimension of " + paddedDimension + " at " + + bits + " bits needs " + paddedBits + " bits per row, more than a quantized matrix " + + "can address; use a smaller dimension"); + } + return (int) ((paddedBits + 7) / 8); + } + /** {@return the number of rows} */ public int rowCount() { return rowCount; @@ -556,7 +576,7 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { } } final int paddedDimension = HadamardRotation.paddedDimension(dimension); - final int rowBytes = (paddedDimension * bits + 7) / 8; + final int rowBytes = rowByteCount(paddedDimension, bits); requireStorableSize(rowCount, rowBytes); final byte[] codes = new byte[rowCount * rowBytes]; try { diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixEdgeCaseTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixEdgeCaseTest.java new file mode 100644 index 0000000000..95c42ed339 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixEdgeCaseTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.util.Random; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Edge cases for the quantized matrix: dimensions that are not powers of two (so the padding + * and truncation path runs), a one-dimensional matrix, adversarial rows the random rotation + * must still reconstruct (constant, one-hot, sign-alternating), and the dot/norm consistency + * over padded dimensions. + */ +class QuantizedEmbeddingMatrixEdgeCaseTest { + + private static final long SEED = 3L; + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3, 5, 17, 100, 300, 513}) + void testDotRotatedMatchesOriginalDotAtEveryDimension(int dimension) { + final Random random = new Random(dimension); + final int rows = 8; + final float[] matrix = new float[rows * dimension]; + for (int i = 0; i < matrix.length; i++) { + matrix[i] = (float) random.nextGaussian(); + } + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, rows, dimension, 4, SEED); + final float[] query = new float[dimension]; + for (int d = 0; d < dimension; d++) { + query[d] = (float) random.nextGaussian(); + } + final float[] rotatedQuery = quantized.rotate(query); + for (int row = 0; row < rows; row++) { + final float[] decoded = quantized.decodeRow(row); + double originalDot = 0; + for (int d = 0; d < dimension; d++) { + originalDot += (double) decoded[d] * query[d]; + } + // The rotation is orthonormal, so scoring in rotated space over the padded coordinates + // must equal the original-space dot with the truncated decoded row. + assertEquals(originalDot, quantized.dotRotated(row, rotatedQuery), + 1e-3 * (1 + Math.abs(originalDot)), + "dot mismatch at dimension " + dimension + ", row " + row); + } + } + + @Test + void testConstantRowReconstructsDespiteBeingSpikyAfterRotation() { + // A constant vector is the worst case for the transform: its rotation concentrates all + // energy in one coordinate, which the grid clamps. The per-row least-squares scale must + // absorb that clamp, so the reconstruction still points the same way. + final int dimension = 300; + final float[] matrix = new float[dimension]; + java.util.Arrays.fill(matrix, 0.7f); + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, 1, dimension, 4, SEED); + assertTrue(cosine(matrix, 0, dimension, quantized.decodeRow(0)) > 0.98, + "a constant row must still reconstruct in direction"); + } + + @Test + void testOneHotAndAlternatingRowsReconstruct() { + final int dimension = 128; + final float[] oneHot = new float[dimension]; + oneHot[7] = 3.5f; + final float[] alternating = new float[dimension]; + for (int d = 0; d < dimension; d++) { + alternating[d] = (d % 2 == 0 ? 1f : -1f); + } + for (final float[] row : new float[][] {oneHot, alternating}) { + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(row, 1, dimension, 4, SEED); + assertTrue(cosine(row, 0, dimension, quantized.decodeRow(0)) > 0.95, + "an adversarial row must reconstruct in direction"); + } + } + + @Test + void testRowNormMatchesDecodedRowAtNonPowerOfTwoDimension() { + final int dimension = 17; + final Random random = new Random(17); + final int rows = 5; + final float[] matrix = new float[rows * dimension]; + for (int i = 0; i < matrix.length; i++) { + matrix[i] = 2f * (float) random.nextGaussian(); + } + final QuantizedEmbeddingMatrix quantized = + QuantizedEmbeddingMatrix.quantize(matrix, rows, dimension, 3, SEED); + for (int row = 0; row < rows; row++) { + final float[] decoded = quantized.decodeRow(row); + double sumOfSquares = 0; + for (final float value : decoded) { + sumOfSquares += (double) value * value; + } + assertEquals(Math.sqrt(sumOfSquares), quantized.rowNorm(row), + 1e-4 * (1 + Math.sqrt(sumOfSquares)), + "rowNorm must equal the decoded row's norm at a padded dimension"); + } + } + + /** + * {@return the cosine between a matrix row and a decoded vector} + * + * @param matrix The flat row-major matrix. + * @param base The row's first index. + * @param dimension The row width. + * @param decoded The decoded row. + */ + private static double cosine(float[] matrix, int base, int dimension, float[] decoded) { + return ModelQuantizer.cosine(matrix, base, dimension, decoded); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java index 53d5accb0c..b560b9e9bb 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java @@ -289,6 +289,26 @@ void testRotatedSpaceAccessorsValidate() { () -> quantized.dotRotated(0, new float[7])); } + @Test + void testReadRejectsADimensionThatOverflowsTheRowByteCount(@TempDir Path directory) + throws IOException { + // A header declaring dimension 2^29 makes paddedDimension*bits overflow a signed int, so + // the per-row byte count goes negative. A corrupt or hostile file must be rejected cleanly, + // not crash the reader with an undocumented NegativeArraySizeException. + final Path file = directory.resolve("matrix.bin"); + QuantizedEmbeddingMatrix.quantize(new float[] {1f, 2f, 3f, 4f}, 1, 4, 4, SEED).write(file); + final byte[] bytes = Files.readAllBytes(file); + // dimension is the third big-endian int: after magic[4] and rowCount[4], at offset 8. + final int overflowingDimension = 1 << 29; + bytes[8] = (byte) (overflowingDimension >>> 24); + bytes[9] = (byte) (overflowingDimension >>> 16); + bytes[10] = (byte) (overflowingDimension >>> 8); + bytes[11] = (byte) overflowingDimension; + final Path patched = directory.resolve("overflow.bin"); + Files.write(patched, bytes); + assertThrows(IllegalArgumentException.class, () -> QuantizedEmbeddingMatrix.read(patched)); + } + @Test void testReadRejectsForeignAndTruncatedFiles(@TempDir Path directory) throws IOException { final Path foreign = directory.resolve("foreign.bin"); From fe18a2d347e8f4523d32856060f60f9c8209fd3e Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:23:09 -0400 Subject: [PATCH 75/82] OPENNLP-1895: Pin the checked loader contract for malformed quantized matrices (failing tests) --- .../embeddings/QuantizedMatrixFormatTest.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java new file mode 100644 index 0000000000..164c97d625 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.InvalidFormatException; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins the loader contract of {@link QuantizedEmbeddingMatrix#read(Path)}: malformed + * file content fails with the checked {@link InvalidFormatException}, never with an + * unchecked exception or an allocation failure. + */ +public class QuantizedMatrixFormatTest { + + /** The on-disk magic of a quantized matrix, as written by the writer. */ + private static final int MAGIC = 0x4F4E5131; + + @Test + void testBadMagicIsRejectedAsFormatError(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("bad-magic.quantized"); + try (DataOutputStream out = out(file)) { + out.writeInt(0xCAFEBABE); + } + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> QuantizedEmbeddingMatrix.read(file)); + assertTrue(e.getMessage().contains("magic"), e.getMessage()); + } + + @Test + void testNegativeRowCountIsRejectedAsFormatError(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("negative-rows.quantized"); + try (DataOutputStream out = out(file)) { + out.writeInt(MAGIC); + out.writeInt(-5); + } + assertThrows(InvalidFormatException.class, () -> QuantizedEmbeddingMatrix.read(file)); + } + + @Test + void testImplausibleRowCountFailsBeforeAllocating(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("huge-rows.quantized"); + try (DataOutputStream out = out(file)) { + out.writeInt(MAGIC); + out.writeInt(Integer.MAX_VALUE); + out.writeInt(8); + out.writeInt(4); + out.writeLong(17L); + out.writeInt(16); + } + assertThrows(InvalidFormatException.class, () -> QuantizedEmbeddingMatrix.read(file)); + } + + @Test + void testUnsupportedBitWidthIsRejectedAsFormatError(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("bad-bits.quantized"); + try (DataOutputStream out = out(file)) { + out.writeInt(MAGIC); + out.writeInt(1); + out.writeInt(8); + out.writeInt(7); + } + assertThrows(InvalidFormatException.class, () -> QuantizedEmbeddingMatrix.read(file)); + } + + private static DataOutputStream out(Path file) throws IOException { + final OutputStream raw = Files.newOutputStream(file); + return new DataOutputStream(raw); + } +} From 21ce369f5982cda09eb42fccf54599766429f1a9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:30:10 -0400 Subject: [PATCH 76/82] OPENNLP-1895: Fail malformed quantized matrices with InvalidFormatException Content errors in read now throw the checked loader exception instead of IllegalArgumentException, with a size plausibility guard before any allocation. The ambiguous matrix source and the quantizer's missing safetensors follow the same contract. The CLI pin includes QuantizeModel, which this branch registers. --- .../opennlp/embeddings/ModelQuantizer.java | 4 ++- .../embeddings/QuantizedEmbeddingMatrix.java | 28 +++++++++++++------ .../QuantizedEmbeddingMatrixTest.java | 6 ++-- .../StaticEmbeddingModelQuantizedTest.java | 8 ++++-- ...eddingModelSentencePieceQuantizedTest.java | 6 ++-- .../opennlp/embeddings/cmdline/CLITest.java | 6 ++-- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java index 7fd04d61ef..81009b4e83 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelQuantizer.java @@ -20,6 +20,8 @@ import java.nio.file.Files; import java.nio.file.Path; +import opennlp.tools.util.InvalidFormatException; + /** * Quantizes a static embedding model directory in place: reads the matrix and optional * per-token weights from the directory's {@code model.safetensors}, quantizes the matrix to the @@ -83,7 +85,7 @@ public static Result quantize(Path modelDirectory, int bits, long seed) throws I } final Path safetensorsFile = modelDirectory.resolve(ModelFileNames.SAFETENSORS); if (!Files.isRegularFile(safetensorsFile)) { - throw new IllegalArgumentException("Model directory " + modelDirectory + " has no " + throw new InvalidFormatException("Model directory " + modelDirectory + " has no " + ModelFileNames.SAFETENSORS + " to quantize"); } final SafetensorsFile tensors = SafetensorsFile.read(safetensorsFile); diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java index 7a8567d663..3130522f46 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java @@ -29,6 +29,7 @@ import java.util.Arrays; import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.util.InvalidFormatException; /** * An embedding matrix quantized to {@code 2}-{@code 4} bits per dimension, following the @@ -509,8 +510,9 @@ public void write(Path file) throws IOException { * * @param file The file to read. Must not be {@code null}. * @return The quantized matrix. - * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or its content is - * not a quantized matrix of a supported version. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null}. + * @throws InvalidFormatException Thrown if the content is not a quantized matrix of a + * supported version or declares implausible sizes. * @throws IOException Thrown if reading fails or the file is truncated. */ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { @@ -521,26 +523,36 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { DataInputStream data = new DataInputStream(new BufferedInputStream(in))) { final int magic = data.readInt(); if (magic != MAGIC) { - throw new IllegalArgumentException(file + " is not a quantized embedding matrix " + throw new InvalidFormatException(file + " is not a quantized embedding matrix " + "(magic 0x" + Integer.toHexString(magic) + ", expected 0x" + Integer.toHexString(MAGIC) + ")"); } final int rowCount = data.readInt(); if (rowCount < 1) { - throw new IllegalArgumentException(file + " declares " + rowCount + " rows; a " + throw new InvalidFormatException(file + " declares " + rowCount + " rows; a " + "quantized matrix has at least 1"); } final int dimension = data.readInt(); if (dimension < 1) { - throw new IllegalArgumentException(file + " declares dimension " + dimension + "; a " + throw new InvalidFormatException(file + " declares dimension " + dimension + "; a " + "quantized matrix's dimension is at least 1"); } + final long fileSize = Files.size(file); + if (rowCount > fileSize || dimension > fileSize) { + throw new InvalidFormatException(file + " declares " + rowCount + " rows and dimension " + + dimension + " but holds only " + fileSize + " bytes"); + } final int bits = data.readInt(); - GaussianQuantizer.requireSupportedBits(bits); + try { + GaussianQuantizer.requireSupportedBits(bits); + } catch (IllegalArgumentException e) { + throw new InvalidFormatException(file + " declares an unsupported bit width: " + + e.getMessage()); + } final long seed = data.readLong(); final int levelCount = data.readInt(); if (levelCount != 1 << bits) { - throw new IllegalArgumentException(file + " declares " + levelCount + " grid levels " + throw new InvalidFormatException(file + " declares " + levelCount + " grid levels " + "for " + bits + " bits; expected " + (1 << bits)); } final float[] levels = new float[levelCount]; @@ -552,7 +564,7 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { for (int row = 0; row < rowCount; row++) { scales[row] = data.readFloat(); if (!Float.isFinite(scales[row])) { - throw new IllegalArgumentException(file + " has a non-finite scale for row " + row + throw new InvalidFormatException(file + " has a non-finite scale for row " + row + ": " + scales[row]); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java index b560b9e9bb..b729e3c1c3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java @@ -27,6 +27,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -306,14 +308,14 @@ void testReadRejectsADimensionThatOverflowsTheRowByteCount(@TempDir Path directo bytes[11] = (byte) overflowingDimension; final Path patched = directory.resolve("overflow.bin"); Files.write(patched, bytes); - assertThrows(IllegalArgumentException.class, () -> QuantizedEmbeddingMatrix.read(patched)); + assertThrows(InvalidFormatException.class, () -> QuantizedEmbeddingMatrix.read(patched)); } @Test void testReadRejectsForeignAndTruncatedFiles(@TempDir Path directory) throws IOException { final Path foreign = directory.resolve("foreign.bin"); Files.write(foreign, new byte[] {1, 2, 3, 4, 5, 6, 7, 8}); - assertThrows(IllegalArgumentException.class, () -> QuantizedEmbeddingMatrix.read(foreign)); + assertThrows(InvalidFormatException.class, () -> QuantizedEmbeddingMatrix.read(foreign)); final Path file = directory.resolve("matrix.bin"); QuantizedEmbeddingMatrix.quantize(testMatrix(), ROWS, DIMENSION, 2, SEED).write(file); diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java index 0e41df7bf6..ba971d5583 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java @@ -29,6 +29,8 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -154,8 +156,8 @@ void testBothMatrixFilesPresentIsRejected(@TempDir Path directory) throws IOExce ModelQuantizer.quantize(directory, 4, SEED); assertTrue(Files.isRegularFile(directory.resolve(ModelFileNames.SAFETENSORS))); assertTrue(Files.isRegularFile(directory.resolve(ModelFileNames.QUANTIZED))); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(directory)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(directory)); assertTrue(e.getMessage().contains("has both"), e.getMessage()); assertTrue(e.getMessage().contains(ModelFileNames.QUANTIZED), e.getMessage()); assertTrue(e.getMessage().contains(ModelFileNames.SAFETENSORS), e.getMessage()); @@ -231,7 +233,7 @@ void testRowCountMismatchFailsLoud(@TempDir Path directory) throws IOException { @Test void testQuantizerRequiresTheSafetensors(@TempDir Path directory) { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> ModelQuantizer.quantize(directory, 4, SEED)); assertTrue(e.getMessage().contains(ModelFileNames.SAFETENSORS), e.getMessage()); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java index 4f43066bda..5373bc986b 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceQuantizedTest.java @@ -24,6 +24,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.InvalidFormatException; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -84,8 +86,8 @@ void testQuantizedSentencePieceRanksLikeTheFloatModel(@TempDir Path directory) void testBothMatrixFilesPresentIsRejected(@TempDir Path directory) throws IOException { fixture.write(directory, DIMENSION, true, SEED); ModelQuantizer.quantize(directory, 4, SEED); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(directory)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(directory)); assertTrue(e.getMessage().contains("has both"), e.getMessage()); } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java index dad82c8b85..c60e90cec0 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/cmdline/CLITest.java @@ -39,12 +39,12 @@ class CLITest { /** {@return the tools the dispatcher registers, as parameterized-test arguments} */ static Stream tools() { - return Stream.of(new AssembleModelTool(), new DistillModelTool()); + return Stream.of(new AssembleModelTool(), new DistillModelTool(), new QuantizeModelTool()); } @Test - void testOffersExactlyTheDistillationCommands() { - assertEquals(Set.of("AssembleModel", "DistillModel"), CLI.getToolNames()); + void testOffersExactlyTheModelCommands() { + assertEquals(Set.of("AssembleModel", "DistillModel", "QuantizeModel"), CLI.getToolNames()); } @Test From c22616f393735b350f367f2a9c5c204beaac44e3 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 9 Aug 2026 08:50:06 -0400 Subject: [PATCH 77/82] OPENNLP-1895: Follow the checked loader contract through the quantized table path The row count disagreement and the wordpiece unknown-token check now throw InvalidFormatException like the rest of the module after the OPENNLP-1877 conversion. --- .../opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java index ba971d5583..250698c218 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelQuantizedTest.java @@ -226,8 +226,8 @@ void testRowCountMismatchFailsLoud(@TempDir Path directory) throws IOException { final List extended = new ArrayList<>(Files.readAllLines(vocabularyFile)); extended.add("straggler"); Files.write(vocabularyFile, extended); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> StaticEmbeddingModel.load(directory)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(directory)); assertTrue(e.getMessage().contains("do not belong to the same model"), e.getMessage()); } From b8d13334e0da3fcf83476aeeb637e6db128dacb0 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:17:01 -0400 Subject: [PATCH 78/82] OPENNLP-1895: Pin InvalidFormatException for every malformed-content path and a size bound before allocation (failing tests) The loader contract says malformed file content fails with the checked InvalidFormatException, but the decoded-norm, pooling-weight, stored-grid, and trailing-byte rejections still throw IllegalArgumentException, and a small hostile file declaring huge dimensions reaches per-row allocation and dies with EOFException instead of a format error. These tests pin the intended behavior and fail against the current reader: - testReadRejectsForeignAndTruncatedFiles now expects InvalidFormatException for trailing bytes (was pinning IllegalArgumentException, the wrong contract) - testDeclaredPayloadBeyondFileSizeFailsBeforeAllocating: a 1.1 MB file declaring 1,000,000 rows of 512 dims at 4 bits must fail fast, before allocating 256 MB of codes plus 8 MB of scales and norms - non-finite stored grid levels, decoded norms, and pooling weights, and a pooling-weight flag with no weights present, must all fail with InvalidFormatException --- .../QuantizedEmbeddingMatrixTest.java | 2 +- .../embeddings/QuantizedMatrixFormatTest.java | 116 ++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java index b729e3c1c3..3cbab13f46 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedEmbeddingMatrixTest.java @@ -327,7 +327,7 @@ void testReadRejectsForeignAndTruncatedFiles(@TempDir Path directory) throws IOE final Path trailing = directory.resolve("trailing.bin"); final byte[] extra = Arrays.copyOf(full, full.length + 1); Files.write(trailing, extra); - assertThrows(IllegalArgumentException.class, () -> QuantizedEmbeddingMatrix.read(trailing)); + assertThrows(InvalidFormatException.class, () -> QuantizedEmbeddingMatrix.read(trailing)); } } diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java index 164c97d625..1b05604702 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/QuantizedMatrixFormatTest.java @@ -21,6 +21,7 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -28,6 +29,7 @@ import opennlp.tools.util.InvalidFormatException; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -87,6 +89,120 @@ void testUnsupportedBitWidthIsRejectedAsFormatError(@TempDir Path dir) throws IO assertThrows(InvalidFormatException.class, () -> QuantizedEmbeddingMatrix.read(file)); } + @Test + void testDeclaredPayloadBeyondFileSizeFailsBeforeAllocating(@TempDir Path dir) + throws IOException { + // A 1.1 MB file declaring 1,000,000 rows of 512 dimensions at 4 bits describes 256 MB of + // packed codes plus 8 MB of scales and norms. Both dimensions individually pass a + // "smaller than the file size" plausibility check, so the loader must hold the declared + // total against the bytes actually present, before allocating anything row-sized. + final Path file = dir.resolve("huge-payload.quantized"); + try (DataOutputStream out = out(file)) { + out.writeInt(MAGIC); + out.writeInt(1_000_000); + out.writeInt(512); + out.writeInt(4); + out.writeLong(17L); + out.writeInt(16); + for (int i = 0; i < 16; i++) { + out.writeFloat(i - 7.5f); + } + out.write(new byte[1_100_000 - 92]); + } + final InvalidFormatException e = assertTimeoutPreemptively(Duration.ofSeconds(10), + () -> assertThrows(InvalidFormatException.class, + () -> QuantizedEmbeddingMatrix.read(file))); + assertTrue(e.getMessage().contains(file.toString()), e.getMessage()); + } + + @Test + void testInvalidStoredGridIsRejectedAsFormatError(@TempDir Path dir) throws IOException { + final Path file = validFile(dir); + final byte[] bytes = Files.readAllBytes(file); + // The first grid level is the big-endian float at offset 28, after the six header fields. + writeNaN(bytes, 28); + final Path patched = dir.resolve("bad-grid.quantized"); + Files.write(patched, bytes); + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> QuantizedEmbeddingMatrix.read(patched)); + assertTrue(e.getMessage().contains("grid"), e.getMessage()); + } + + @Test + void testNonFiniteDecodedNormIsRejectedAsFormatError(@TempDir Path dir) throws IOException { + final Path file = validFile(dir); + final byte[] bytes = Files.readAllBytes(file); + // Row 0's decoded norm is the big-endian float at offset 96: 28 header bytes, 64 bytes of + // grid levels, and 4 bytes for row 0's scale. + writeNaN(bytes, 96); + final Path patched = dir.resolve("bad-norm.quantized"); + Files.write(patched, bytes); + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> QuantizedEmbeddingMatrix.read(patched)); + assertTrue(e.getMessage().contains("norm"), e.getMessage()); + } + + @Test + void testNonFinitePoolingWeightIsRejectedAsFormatError(@TempDir Path dir) throws IOException { + final Path file = dir.resolve("valid-weights.quantized"); + QuantizedEmbeddingMatrix.quantize(new float[] {1f, 2f, 3f, 4f}, 1, 4, 4, 17L) + .withPoolingWeights(new float[] {2f}) + .write(file); + final byte[] bytes = Files.readAllBytes(file); + // Row 0's pooling weight is the big-endian float at offset 101, right after the weight + // presence flag at offset 100. + writeNaN(bytes, 101); + final Path patched = dir.resolve("bad-weight.quantized"); + Files.write(patched, bytes); + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> QuantizedEmbeddingMatrix.read(patched)); + assertTrue(e.getMessage().contains("pooling"), e.getMessage()); + } + + @Test + void testPoolingWeightsDeclaredBeyondFileSizeAreRejectedAsFormatError(@TempDir Path dir) + throws IOException { + final Path file = validFile(dir); + final byte[] bytes = Files.readAllBytes(file); + // Flipping the presence flag at offset 100 declares per-row pooling weights the file does + // not contain, so the declared total exceeds the file size. + bytes[100] = 1; + final Path patched = dir.resolve("flagged-weights.quantized"); + Files.write(patched, bytes); + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> QuantizedEmbeddingMatrix.read(patched)); + assertTrue(e.getMessage().contains("pooling"), e.getMessage()); + } + + /** + * {@return a valid one-row, four-dimension, 4-bit quantized file without pooling weights} + * Its layout is fixed: 28 header bytes, 64 bytes of grid levels, one scale at offset 92, one + * decoded norm at offset 96, the weight presence flag at offset 100, and two packed code + * bytes, 103 bytes in total. + * + * @param dir The directory to write into. + * @throws IOException Thrown if writing fails. + */ + private static Path validFile(Path dir) throws IOException { + final Path file = dir.resolve("valid.quantized"); + QuantizedEmbeddingMatrix.quantize(new float[] {1f, 2f, 3f, 4f}, 1, 4, 4, 17L).write(file); + return file; + } + + /** + * Overwrites four bytes with the big-endian bits of {@code Float.NaN}. + * + * @param bytes The file image to patch. + * @param offset The offset of the float to replace. + */ + private static void writeNaN(byte[] bytes, int offset) { + final int nan = Float.floatToIntBits(Float.NaN); + bytes[offset] = (byte) (nan >>> 24); + bytes[offset + 1] = (byte) (nan >>> 16); + bytes[offset + 2] = (byte) (nan >>> 8); + bytes[offset + 3] = (byte) nan; + } + private static DataOutputStream out(Path file) throws IOException { final OutputStream raw = Files.newOutputStream(file); return new DataOutputStream(raw); From c45b8df4c7ad23ab2da066d596ad50a6f525c9d0 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:18:25 -0400 Subject: [PATCH 79/82] OPENNLP-1895: Fail every malformed-content path with InvalidFormatException and bound declared sizes before allocating read(Path) promises the checked InvalidFormatException for malformed content, but four rejections still threw IllegalArgumentException and escaped every catch (IOException): an invalid decoded norm, a non-finite pooling weight, trailing bytes after the declared content, and the row-byte-count and storable-size checks reached from a hostile header. A stored grid that fromLevels rejects also surfaced as IllegalArgumentException. All of these now throw InvalidFormatException naming the file and the offending field; constructor validation for programmatic callers stays IllegalArgumentException. The reader also only checked rowCount and dimension individually against the file size, so a 1.1 MB file declaring 1,000,000 rows of 512 dims at 4 bits forced a 256 MB code allocation before any content check. The header fully determines the file size, so the declared total (28 fixed bytes, the grid levels, a scale and a decoded norm per row, the flag byte, and the packed codes, plus the per-row pooling weights when flagged) is now held against the actual file size and rejected before anything row-sized is allocated. --- .../embeddings/QuantizedEmbeddingMatrix.java | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java index 3130522f46..e6fcc82466 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java @@ -511,8 +511,10 @@ public void write(Path file) throws IOException { * @param file The file to read. Must not be {@code null}. * @return The quantized matrix. * @throws IllegalArgumentException Thrown if {@code file} is {@code null}. - * @throws InvalidFormatException Thrown if the content is not a quantized matrix of a - * supported version or declares implausible sizes. + * @throws InvalidFormatException Thrown if the content is malformed: not a quantized matrix + * of a supported version, declaring sizes the file's bytes cannot back or this reader + * cannot store, storing an invalid grid or non-finite per-row values, or carrying + * trailing bytes after the declared content. * @throws IOException Thrown if reading fails or the file is truncated. */ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { @@ -555,11 +557,37 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { throw new InvalidFormatException(file + " declares " + levelCount + " grid levels " + "for " + bits + " bits; expected " + (1 << bits)); } + final int rowBytes; + try { + rowBytes = rowByteCount(HadamardRotation.paddedDimension(dimension), bits); + requireStorableSize(rowCount, rowBytes); + } catch (IllegalArgumentException e) { + throw new InvalidFormatException(file + " declares a matrix this reader cannot " + + "store: " + e.getMessage()); + } + // The header fully determines the file size: 28 fixed bytes (magic, row count, dimension, + // and bit width at 4 each, the 8-byte seed, the 4-byte level count), the grid levels, a + // scale and a decoded norm per row, the 1-byte pooling-weight flag, and the packed codes. + // Everything past this point allocates per-row storage, so the declared total is held + // against the bytes actually present first: a small hostile file must not force a giant + // allocation before its content is ever read. + final long declaredBytes = 28L + 4L * levelCount + (8L + rowBytes) * rowCount + 1L; + if (declaredBytes > fileSize) { + throw new InvalidFormatException(file + " declares " + rowCount + " rows and " + + "dimension " + dimension + " at " + bits + " bits, needing at least " + + declaredBytes + " bytes of scales, norms, and packed codes, but holds only " + + fileSize + " bytes"); + } final float[] levels = new float[levelCount]; for (int i = 0; i < levelCount; i++) { levels[i] = data.readFloat(); } - final GaussianQuantizer quantizer = GaussianQuantizer.fromLevels(levels); + final GaussianQuantizer quantizer; + try { + quantizer = GaussianQuantizer.fromLevels(levels); + } catch (IllegalArgumentException e) { + throw new InvalidFormatException(file + " stores an invalid grid: " + e.getMessage()); + } final float[] scales = new float[rowCount]; for (int row = 0; row < rowCount; row++) { scales[row] = data.readFloat(); @@ -572,24 +600,26 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { for (int row = 0; row < rowCount; row++) { decodedNorms[row] = data.readFloat(); if (!Float.isFinite(decodedNorms[row]) || decodedNorms[row] < 0) { - throw new IllegalArgumentException(file + " has an invalid decoded norm for row " + throw new InvalidFormatException(file + " has an invalid decoded norm for row " + row + ": " + decodedNorms[row]); } } float[] poolingWeights = null; if (data.readBoolean()) { + if (declaredBytes + 4L * rowCount > fileSize) { + throw new InvalidFormatException(file + " declares per-row pooling weights, " + + "needing at least " + (declaredBytes + 4L * rowCount) + " bytes in total, but " + + "holds only " + fileSize + " bytes"); + } poolingWeights = new float[rowCount]; for (int row = 0; row < rowCount; row++) { poolingWeights[row] = data.readFloat(); if (!Float.isFinite(poolingWeights[row])) { - throw new IllegalArgumentException(file + " has a non-finite pooling weight for " + throw new InvalidFormatException(file + " has a non-finite pooling weight for " + "row " + row + ": " + poolingWeights[row]); } } } - final int paddedDimension = HadamardRotation.paddedDimension(dimension); - final int rowBytes = rowByteCount(paddedDimension, bits); - requireStorableSize(rowCount, rowBytes); final byte[] codes = new byte[rowCount * rowBytes]; try { data.readFully(codes); @@ -598,7 +628,7 @@ public static QuantizedEmbeddingMatrix read(Path file) throws IOException { + " rows of " + rowBytes + " packed bytes, but the file ends early", e); } if (data.read() != -1) { - throw new IllegalArgumentException(file + " has trailing bytes after the declared " + throw new InvalidFormatException(file + " has trailing bytes after the declared " + "content; it is not a quantized matrix of this version"); } return new QuantizedEmbeddingMatrix(rowCount, dimension, bits, seed, quantizer, scales, From ded6805a9edf70ce1ef3aa62bc25a35c5e0e4ae3 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 10 Aug 2026 01:19:43 -0400 Subject: [PATCH 80/82] OPENNLP-1895: Correct the quantized size numbers for power-of-two padding The class comment and the manual claimed a 500,000-row, 300-dimension table drops to 77 MB at 4 bits. The Hadamard rotation pads rows to the next power of two, so 300 dimensions store 512 codes per row, and each row also carries two floats (the fitted scale and the decoded norm), not one. Measured from a written file, that is 264 bytes per row: 132 MB against 600 MB of float32, 4.5 times smaller, not 7.8. The overall shrink range becomes roughly 4 to 16 times depending on bit width and padding distance. The padding behavior itself is unchanged. --- opennlp-docs/src/docbkx/embeddings.xml | 7 +++++-- .../opennlp/embeddings/QuantizedEmbeddingMatrix.java | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 488739608d..34a24ea865 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -228,8 +228,11 @@ results.sort(Comparator.comparingDouble(Scored::score).reversed());]]> Quantized Models A static embedding table can be quantized to 2, 3, or 4 bits per dimension, shrinking - it by roughly 8 to 16 times against the 32-bit float matrix (a 500,000-row, - 300-dimension table drops from about 600 MB to 77 MB at 4 bits). Because embedding is + it by roughly 4 to 16 times against the 32-bit float matrix, depending on the bit + width and on how far the dimension is from a power of two: rows are padded to the + next power of two before coding, so a 500,000-row, 300-dimension table stores 512 + coded dimensions per row and drops from about 600 MB to about 132 MB at 4 bits, 4.5 + times smaller. Because embedding is memory-bound row gathering, reading fewer bytes is also the throughput lever. The method is the TurboQuant construction: each row is rotated so its coordinates become near-independent and near-Gaussian, and each rotated coordinate is encoded against an diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java index e6fcc82466..5019a9ffc3 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/QuantizedEmbeddingMatrix.java @@ -42,10 +42,12 @@ * its grid levels; the scale is least-squares fitted per row, which strictly reduces the squared * error of the fixed grid. * - *

The storage is {@code bits} per dimension plus one float per row, against 32 bits per - * dimension for the float matrix: a 500,000-row, 300-dimension table shrinks from roughly 600 MB - * to 77 MB at 4 bits (the padded dimension, 512 here, is what is stored). The workload this - * serves is memory-bound row gathering, so reading fewer bytes is also the throughput lever.

+ *

The storage is {@code bits} per padded dimension plus two floats per row (the + * fitted scale and the decoded norm), against 32 bits per dimension for the float matrix. The + * rotation pads each row to the next power of two, so a 500,000-row, 300-dimension table stores + * 512 coded dimensions per row and shrinks from roughly 600 MB to about 132 MB at 4 bits, 4.5 + * times smaller. The workload this serves is memory-bound row gathering, so reading fewer bytes + * is also the throughput lever.

* *

Rows live in rotated space, and the cheap operations stay there: the rotation is * orthonormal, so dot products and norms of rotated vectors equal those of the originals, and From f6f1bef48e4da4a2c12d682be5c1f316885e7af9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 21 Aug 2026 16:09:07 -0400 Subject: [PATCH 81/82] OPENNLP-1877: load self-contained Model2Vec Unigram tokenizers Red evidence: Model2VecUnigramTokenizerTest failed because missing model.vocab leaked a NullPointerException, and StaticEmbeddingModelSentencePieceTest failed because an incomplete legacy tokenizer did not explain the supported alternatives. The loader now reconstructs the published Unigram tokenizer in memory from tokenizer.json, including its precompiled normalizer and supported Model2Vec post-normalization steps. The pinned potion-multilingual-128M artifact loads without a borrowed teacher model and matches the reference Python vector. --- opennlp-docs/src/docbkx/embeddings.xml | 19 +- .../java/opennlp/embeddings/JsonCursor.java | 37 + .../embeddings/Model2VecUnigramTokenizer.java | 680 ++++++++++++++++++ .../opennlp/embeddings/ModelAssembler.java | 44 +- .../opennlp/embeddings/ModelFileNames.java | 5 +- .../embeddings/StaticEmbeddingModel.java | 67 +- .../embeddings/cmdline/AssembleModelTool.java | 7 +- .../Model2VecUnigramTokenizerTest.java | 47 ++ .../embeddings/ModelAssemblerTest.java | 38 +- ...StaticEmbeddingModelSentencePieceTest.java | 6 +- 10 files changed, 879 insertions(+), 71 deletions(-) create mode 100644 opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Model2VecUnigramTokenizer.java create mode 100644 opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/Model2VecUnigramTokenizerTest.java diff --git a/opennlp-docs/src/docbkx/embeddings.xml b/opennlp-docs/src/docbkx/embeddings.xml index 69fd85d1b9..12b7a785cc 100644 --- a/opennlp-docs/src/docbkx/embeddings.xml +++ b/opennlp-docs/src/docbkx/embeddings.xml @@ -64,12 +64,12 @@ A model directory loads with a single call, and the tokenizer family is detected from the files present. A WordPiece model carries vocab.txt, model.safetensors, config.json, and - tokenizer_config.json; a SentencePiece model carries a trained - .model file (sentencepiece.bpe.model, - spiece.model, or tokenizer.model) next to + tokenizer_config.json; a Model2Vec Unigram model carries tokenizer.json, model.safetensors, and - config.json. The tokenizer and pooling switches are read from the model's - own configuration files: + config.json. Its JSON file contains the Unigram vocabulary, scores, + normalizer, and pre-tokenizer. Legacy SentencePiece directories may also carry a + trained .model file. The tokenizer and pooling switches are read from the + model's own configuration files: Matrix rows are resolved by piece string, never by tokenizer id, because the two - files of a SentencePiece model routinely order and offset their ids differently. A - poolable piece with no matrix row fails loud at load time. Distillation output - usually ships without the trained .model file; copy that one file from - the teacher model's own repository into the model directory, and the loader names - exactly this fix if the file is missing. + files of a legacy SentencePiece model may order and offset their ids differently. A + poolable piece with no matrix row fails loud at load time. Current Model2Vec Unigram + releases load directly from their self-contained tokenizer.json and do not + require a tokenizer file copied from the teacher. Instances are immutable and safe for concurrent use, so one loaded model can serve diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java index 236a9aed0a..1300fcd73d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/JsonCursor.java @@ -249,6 +249,43 @@ long parseLong() throws InvalidFormatException { } } + /** + * {@return the finite JSON number starting at the cursor, parsed as a {@code double}} + * + * @throws InvalidFormatException Thrown if no JSON number is present or its value is not + * finite. + */ + double parseDouble() throws InvalidFormatException { + final int start = position; + skipNumber(); + final String number = text.substring(start, position); + try { + final double value = Double.parseDouble(number); + if (!Double.isFinite(value)) { + throw malformed("Number is not finite: " + number); + } + return value; + } catch (NumberFormatException e) { + throw malformed("Malformed number: " + number); + } + } + + /** + * {@return the JSON boolean starting at the cursor} + * + * @throws InvalidFormatException Thrown if the next value is not {@code true} or + * {@code false}. + */ + boolean parseBoolean() throws InvalidFormatException { + if (consumeLiteral("true")) { + return true; + } + if (consumeLiteral("false")) { + return false; + } + throw malformed("Expected a boolean"); + } + /** * Skips one JSON value of any type (string, number, array, object, true/false/null), so a * reader tolerates fields it does not care about. diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Model2VecUnigramTokenizer.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Model2VecUnigramTokenizer.java new file mode 100644 index 0000000000..0db1734af5 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/Model2VecUnigramTokenizer.java @@ -0,0 +1,680 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import opennlp.subword.sentencepiece.SentencePieceTokenizer; +import opennlp.tools.tokenize.SubwordPiece; +import opennlp.tools.tokenize.SubwordTokenizer; +import opennlp.tools.util.InvalidFormatException; + +/** + * Runs the Unigram tokenizer stored directly in a Model2Vec {@code tokenizer.json}. + * + *

Hugging Face stores the pieces and their scores in JSON, while OpenNLP's pure Java Unigram + * decoder reads the equivalent SentencePiece protobuf. This adapter creates that protobuf in + * memory. A tiny second tokenizer runs the JSON file's precompiled character map before the + * supported post-normalization steps are applied. No generated tokenizer file is written beside + * the user-supplied model.

+ */ +final class Model2VecUnigramTokenizer implements SubwordTokenizer { + + private static final int TYPE_NORMAL = 1; + private static final int TYPE_UNKNOWN = 2; + private static final int TYPE_CONTROL = 3; + private static final int TYPE_BYTE = 6; + private static final String UNIGRAM = "Unigram"; + private static final String METASPACE = "Metaspace"; + private static final String SEQUENCE = "Sequence"; + private static final String PRECOMPILED = "Precompiled"; + private static final String REPLACE = "Replace"; + private static final String STRIP = "Strip"; + private static final String MARKER = "▁"; + private static final char MARKER_CHAR = '▁'; + + private final SentencePieceTokenizer normalizer; + private final SentencePieceTokenizer segmenter; + private final List operations; + private final int unknownId; + private final Set controlIds; + + private Model2VecUnigramTokenizer(Parsed parsed) throws IOException { + final byte[] normalizerModel = modelBytes( + List.of(new Piece("", 0f, TYPE_UNKNOWN)), 0, false, + parsed.precompiledCharsMap(), true); + normalizer = SentencePieceTokenizer.load(new ByteArrayInputStream(normalizerModel)); + segmenter = SentencePieceTokenizer.load(new ByteArrayInputStream(modelBytes( + parsed.pieces(), parsed.unknownId(), parsed.byteFallback(), new byte[0], false))); + operations = List.copyOf(parsed.operations()); + unknownId = parsed.unknownId(); + controlIds = Set.copyOf(parsed.controlIds()); + } + + /** Reads and validates a supported Model2Vec Unigram tokenizer. */ + static Model2VecUnigramTokenizer load(Path tokenizerJson) throws IOException { + if (tokenizerJson == null) { + throw new IllegalArgumentException("tokenizerJson must not be null"); + } + if (!Files.isRegularFile(tokenizerJson)) { + throw new IllegalArgumentException( + "File does not exist or is not a regular file: " + tokenizerJson); + } + return new Model2VecUnigramTokenizer(parse(tokenizerJson)); + } + + /** {@inheritDoc} */ + @Override + public List encode(CharSequence text) { + if (text == null) { + throw new IllegalArgumentException("text must not be null"); + } + String normalized = normalizer.normalize(text).toString(); + for (NormalizationOperation operation : operations) { + normalized = operation.apply(normalized); + } + return segmenter.encode(normalized); + } + + /** {@return whether the row is the tokenizer's unknown piece} */ + boolean isUnknown(int id) { + return id == unknownId; + } + + /** {@return whether the row is a special control piece} */ + boolean isControl(int id) { + return controlIds.contains(id); + } + + /** {@return the number of tokenizer rows} */ + int vocabularySize() { + return segmenter.vocabularySize(); + } + + /** {@return the piece at the given tokenizer row} */ + String idToPiece(int id) { + return segmenter.idToPiece(id); + } + + private static Parsed parse(Path file) throws IOException { + final JsonCursor cursor = new JsonCursor(Files.readString(file), file.getFileName().toString()); + cursor.skipWhitespace(); + cursor.expect('{'); + cursor.skipWhitespace(); + ParsedModel model = null; + ParsedNormalizer normalizer = null; + boolean metaspace = false; + List addedTokens = List.of(); + if (cursor.peek() != '}') { + while (true) { + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "model" -> model = parseModel(cursor); + case "normalizer" -> normalizer = parseNormalizer(cursor); + case "pre_tokenizer" -> metaspace = parsePreTokenizer(cursor); + case "added_tokens" -> addedTokens = parseAddedTokens(cursor); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + continue; + } + if (next == '}') { + break; + } + throw cursor.malformed("Expected ',' or '}' after a top-level field"); + } + } else { + cursor.consume(); + } + cursor.requireEnd("Trailing content after the top-level object"); + if (model == null || !UNIGRAM.equals(model.type())) { + throw new InvalidFormatException(file + " does not define a Unigram tokenizer model"); + } + if (model.pieces() == null || model.pieces().isEmpty()) { + throw new InvalidFormatException(file + " has no model.vocab entries"); + } + if (normalizer == null || normalizer.precompiledCharsMap() == null) { + throw new InvalidFormatException(file + " has no supported Precompiled normalizer"); + } + if (!metaspace) { + throw new InvalidFormatException(file + " has no supported Metaspace pre-tokenizer"); + } + final List pieces = new ArrayList<>(model.pieces()); + final Set controls = new HashSet<>(); + final List sorted = new ArrayList<>(addedTokens); + sorted.sort(Comparator.comparingInt(AddedToken::id)); + for (AddedToken added : sorted) { + if (added.id() >= pieces.size()) { + throw new InvalidFormatException(file + " declares added token id " + added.id() + + " outside the model vocabulary of " + pieces.size() + " rows"); + } + if (!pieces.get(added.id()).text().equals(added.content())) { + throw new InvalidFormatException(file + " contradicts model.vocab at added token id " + + added.id()); + } + if (added.special() && added.id() != model.unknownId()) { + controls.add(added.id()); + } + } + if (model.unknownId() < 0 || model.unknownId() >= pieces.size()) { + throw new InvalidFormatException(file + " has an invalid model.unk_id"); + } + for (int id = 0; id < pieces.size(); id++) { + final Piece piece = pieces.get(id); + final int type = id == model.unknownId() ? TYPE_UNKNOWN + : controls.contains(id) ? TYPE_CONTROL + : model.byteFallback() && isBytePiece(piece.text()) ? TYPE_BYTE : TYPE_NORMAL; + pieces.set(id, new Piece(piece.text(), piece.score(), type)); + } + return new Parsed(pieces, model.unknownId(), model.byteFallback(), + normalizer.precompiledCharsMap(), normalizer.operations(), controls); + } + + private static ParsedModel parseModel(JsonCursor cursor) throws InvalidFormatException { + cursor.expect('{'); + cursor.skipWhitespace(); + String type = null; + int unknownId = -1; + boolean byteFallback = false; + List pieces = null; + while (cursor.peek() != '}') { + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "type" -> type = cursor.parseString(); + case "unk_id" -> unknownId = checkedInt(cursor.parseLong(), cursor, "model.unk_id"); + case "byte_fallback" -> byteFallback = cursor.parseBoolean(); + case "vocab" -> pieces = parseVocabulary(cursor); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + } else if (next != '}') { + throw cursor.malformed("Expected ',' or '}' after a model field"); + } else { + return new ParsedModel(type, unknownId, byteFallback, pieces); + } + } + cursor.consume(); + return new ParsedModel(type, unknownId, byteFallback, pieces); + } + + private static List parseVocabulary(JsonCursor cursor) throws InvalidFormatException { + cursor.expect('['); + cursor.skipWhitespace(); + final List pieces = new ArrayList<>(); + while (cursor.peek() != ']') { + cursor.expect('['); + cursor.skipWhitespace(); + final String text = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(','); + cursor.skipWhitespace(); + final double score = cursor.parseDouble(); + if (score < -Float.MAX_VALUE || score > Float.MAX_VALUE) { + throw cursor.malformed("Unigram score is outside the float range"); + } + cursor.skipWhitespace(); + cursor.expect(']'); + pieces.add(new Piece(text, (float) score, TYPE_NORMAL)); + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + } else if (next != ']') { + throw cursor.malformed("Expected ',' or ']' after a vocabulary entry"); + } else { + return pieces; + } + } + cursor.consume(); + return pieces; + } + + private static ParsedNormalizer parseNormalizer(JsonCursor cursor) + throws InvalidFormatException { + final NormalizerBuilder builder = new NormalizerBuilder(); + parseNormalizerObject(cursor, builder); + return new ParsedNormalizer(builder.precompiledCharsMap, builder.operations); + } + + private static void parseNormalizerObject(JsonCursor cursor, NormalizerBuilder builder) + throws InvalidFormatException { + cursor.expect('{'); + cursor.skipWhitespace(); + String type = null; + String precompiled = null; + List children = null; + PatternValue pattern = null; + String content = null; + boolean stripLeft = false; + boolean stripRight = false; + while (cursor.peek() != '}') { + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "type" -> type = cursor.parseString(); + case "precompiled_charsmap" -> precompiled = cursor.parseString(); + case "normalizers" -> children = parseNormalizerChildren(cursor); + case "pattern" -> pattern = parsePattern(cursor); + case "content" -> content = cursor.parseString(); + case "strip_left" -> stripLeft = cursor.parseBoolean(); + case "strip_right" -> stripRight = cursor.parseBoolean(); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + } else if (next != '}') { + throw cursor.malformed("Expected ',' or '}' after a normalizer field"); + } else { + break; + } + } + if (type == null) { + throw cursor.malformed("Normalizer has no type"); + } + switch (type) { + case SEQUENCE -> { + if (children == null) { + throw cursor.malformed("Sequence normalizer has no normalizers list"); + } + for (ParsedNormalizer child : children) { + if (child.precompiledCharsMap() != null) { + if (builder.precompiledCharsMap != null) { + throw cursor.malformed("More than one Precompiled normalizer is not supported"); + } + builder.precompiledCharsMap = child.precompiledCharsMap(); + } + builder.operations.addAll(child.operations()); + } + } + case PRECOMPILED -> { + if (precompiled == null) { + throw cursor.malformed("Precompiled normalizer has no character map"); + } + try { + builder.precompiledCharsMap = Base64.getDecoder().decode(precompiled); + } catch (IllegalArgumentException e) { + throw cursor.malformed("Precompiled normalizer has malformed base64"); + } + } + case REPLACE -> builder.operations.add(replacement(pattern, content, cursor)); + case STRIP -> builder.operations.add(new StripOperation(stripLeft, stripRight)); + default -> throw cursor.malformed("Unsupported normalizer type '" + type + "'"); + } + } + + private static List parseNormalizerChildren(JsonCursor cursor) + throws InvalidFormatException { + cursor.expect('['); + cursor.skipWhitespace(); + final List children = new ArrayList<>(); + while (cursor.peek() != ']') { + final NormalizerBuilder child = new NormalizerBuilder(); + parseNormalizerObject(cursor, child); + children.add(new ParsedNormalizer(child.precompiledCharsMap, child.operations)); + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + } else if (next != ']') { + throw cursor.malformed("Expected ',' or ']' after a normalizer"); + } else { + return children; + } + } + cursor.consume(); + return children; + } + + private static PatternValue parsePattern(JsonCursor cursor) throws InvalidFormatException { + cursor.expect('{'); + cursor.skipWhitespace(); + final String kind = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + final String value = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect('}'); + return new PatternValue(kind, value); + } + + private static NormalizationOperation replacement( + PatternValue pattern, String content, JsonCursor cursor) throws InvalidFormatException { + if (pattern == null || content == null) { + throw cursor.malformed("Replace normalizer needs pattern and content"); + } + if ("String".equals(pattern.kind())) { + if (!content.equals(" " + pattern.value() + " ")) { + throw cursor.malformed("Only spacing literal replacements are supported"); + } + return new SurroundOperation(pattern.value()); + } + if ("Regex".equals(pattern.kind()) + && ("\\s+".equals(pattern.value()) || " {2,}".equals(pattern.value())) + && " ".equals(content)) { + return CollapseOperation.INSTANCE; + } + throw cursor.malformed("Unsupported Replace normalizer pattern"); + } + + private static boolean parsePreTokenizer(JsonCursor cursor) throws InvalidFormatException { + cursor.expect('{'); + cursor.skipWhitespace(); + String type = null; + String replacement = null; + String prependScheme = null; + boolean split = true; + while (cursor.peek() != '}') { + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "type" -> type = cursor.parseString(); + case "replacement" -> replacement = cursor.parseString(); + case "prepend_scheme" -> prependScheme = cursor.parseString(); + case "split" -> split = cursor.parseBoolean(); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + } else if (next != '}') { + throw cursor.malformed("Expected ',' or '}' after a pre-tokenizer field"); + } else { + break; + } + } + return METASPACE.equals(type) && MARKER.equals(replacement) + && "always".equals(prependScheme) && !split; + } + + private static List parseAddedTokens(JsonCursor cursor) + throws InvalidFormatException { + cursor.expect('['); + cursor.skipWhitespace(); + final List tokens = new ArrayList<>(); + while (cursor.peek() != ']') { + cursor.expect('{'); + cursor.skipWhitespace(); + int id = -1; + String content = null; + boolean special = false; + while (cursor.peek() != '}') { + final String key = cursor.parseString(); + cursor.skipWhitespace(); + cursor.expect(':'); + cursor.skipWhitespace(); + switch (key) { + case "id" -> id = checkedInt(cursor.parseLong(), cursor, "added token id"); + case "content" -> content = cursor.parseString(); + case "special" -> special = cursor.parseBoolean(); + default -> cursor.skipValue(); + } + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + } else if (next != '}') { + throw cursor.malformed("Expected ',' or '}' after an added-token field"); + } else { + break; + } + } + if (id < 0 || content == null) { + throw cursor.malformed("Added token needs id and content"); + } + tokens.add(new AddedToken(id, content, special)); + cursor.skipWhitespace(); + final char next = cursor.consume(); + if (next == ',') { + cursor.skipWhitespace(); + } else if (next != ']') { + throw cursor.malformed("Expected ',' or ']' after an added token"); + } else { + return tokens; + } + } + cursor.consume(); + return tokens; + } + + private static int checkedInt(long value, JsonCursor cursor, String field) + throws InvalidFormatException { + if (value < 0 || value > Integer.MAX_VALUE) { + throw cursor.malformed(field + " is outside the supported range"); + } + return (int) value; + } + + private static boolean isBytePiece(String piece) { + if (piece.length() != 6 || piece.charAt(0) != '<' || piece.charAt(1) != '0' + || piece.charAt(2) != 'x' || piece.charAt(5) != '>') { + return false; + } + return Character.digit(piece.charAt(3), 16) >= 0 + && Character.digit(piece.charAt(4), 16) >= 0; + } + + private static byte[] modelBytes(List pieces, int unknownId, boolean byteFallback, + byte[] precompiledCharsMap, boolean normalizing) { + final ProtoWriter model = new ProtoWriter(); + for (int id = 0; id < pieces.size(); id++) { + final Piece piece = pieces.get(id); + final ProtoWriter entry = new ProtoWriter(); + entry.string(1, piece.text()); + entry.float32(2, piece.score()); + entry.varintField(3, id == unknownId ? TYPE_UNKNOWN : piece.type()); + model.message(1, entry.bytes()); + } + final ProtoWriter trainer = new ProtoWriter(); + trainer.varintField(3, 1); + if (byteFallback) { + trainer.varintField(35, 1); + } + model.message(2, trainer.bytes()); + final ProtoWriter normalizer = new ProtoWriter(); + if (precompiledCharsMap.length > 0) { + normalizer.bytesField(2, precompiledCharsMap); + } + normalizer.varintField(3, normalizing ? 1 : 0); + normalizer.varintField(4, normalizing ? 1 : 0); + normalizer.varintField(5, normalizing ? 1 : 0); + model.message(3, normalizer.bytes()); + return model.bytes(); + } + + private interface NormalizationOperation { + String apply(String input); + } + + private record SurroundOperation(String literal) implements NormalizationOperation { + @Override + public String apply(String input) { + if (literal.isEmpty()) { + return input; + } + final StringBuilder out = new StringBuilder(input.length() + 8); + int cursor = 0; + while (cursor < input.length()) { + if (input.startsWith(literal, cursor)) { + appendMarker(out); + out.append(literal); + appendMarker(out); + cursor += literal.length(); + } else { + final int codePoint = input.codePointAt(cursor); + out.appendCodePoint(codePoint); + cursor += Character.charCount(codePoint); + } + } + return out.toString(); + } + } + + private enum CollapseOperation implements NormalizationOperation { + INSTANCE; + + @Override + public String apply(String input) { + final StringBuilder out = new StringBuilder(input.length()); + boolean marker = false; + for (int cursor = 0; cursor < input.length(); ) { + final int codePoint = input.codePointAt(cursor); + cursor += Character.charCount(codePoint); + if (codePoint == MARKER_CHAR) { + if (!marker) { + out.append(MARKER_CHAR); + } + marker = true; + } else { + out.appendCodePoint(codePoint); + marker = false; + } + } + return out.toString(); + } + } + + private record StripOperation(boolean left, boolean right) implements NormalizationOperation { + @Override + public String apply(String input) { + int start = 0; + int end = input.length(); + if (right) { + while (end > start && input.charAt(end - 1) == MARKER_CHAR) { + end--; + } + } + if (left) { + while (start < end && input.charAt(start) == MARKER_CHAR) { + start++; + } + if (start > 0 && start < end) { + start--; + } + } + return start == 0 && end == input.length() ? input : input.substring(start, end); + } + } + + private static void appendMarker(StringBuilder out) { + if (out.isEmpty() || out.charAt(out.length() - 1) != MARKER_CHAR) { + out.append(MARKER_CHAR); + } + } + + private record Piece(String text, float score, int type) { + } + + private record ParsedModel( + String type, int unknownId, boolean byteFallback, List pieces) { + } + + private record ParsedNormalizer( + byte[] precompiledCharsMap, List operations) { + } + + private record PatternValue(String kind, String value) { + } + + private record AddedToken(int id, String content, boolean special) { + } + + private record Parsed(List pieces, int unknownId, boolean byteFallback, + byte[] precompiledCharsMap, List operations, + Set controlIds) { + } + + private static final class NormalizerBuilder { + private byte[] precompiledCharsMap; + private final List operations = new ArrayList<>(); + } + + private static final class ProtoWriter { + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + void message(int field, byte[] value) { + bytesField(field, value); + } + + void string(int field, String value) { + bytesField(field, value.getBytes(StandardCharsets.UTF_8)); + } + + void bytesField(int field, byte[] value) { + varint((long) field << 3 | 2); + varint(value.length); + out.writeBytes(value); + } + + void varintField(int field, long value) { + varint((long) field << 3); + varint(value); + } + + void float32(int field, float value) { + varint((long) field << 3 | 5); + final int bits = Float.floatToIntBits(value); + out.write(bits & 0xff); + out.write(bits >>> 8 & 0xff); + out.write(bits >>> 16 & 0xff); + out.write(bits >>> 24 & 0xff); + } + + void varint(long value) { + long remaining = value; + while ((remaining & ~0x7fL) != 0) { + out.write((int) (remaining & 0x7f) | 0x80); + remaining >>>= 7; + } + out.write((int) remaining); + } + + byte[] bytes() { + return out.toByteArray(); + } + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java index a4e64672ee..e4a436908d 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelAssembler.java @@ -37,11 +37,10 @@ * *

A distillation ships {@code model.safetensors}, {@code tokenizer.json}, and * {@code config.json}, but not the two files the loader also needs for a WordPiece model - * ({@code vocab.txt} and {@code tokenizer_config.json}), and not the trained SentencePiece - * {@code .model} file. This class fills the WordPiece gap from {@code tokenizer.json} itself: the - * matrix row order is the {@code model.vocab} dictionary in id order, and the casing is the - * {@code normalizer.lowercase} flag. It cannot fabricate the SentencePiece {@code .model} file, - * which comes from the teacher, so it reports that as an actionable error.

+ * ({@code vocab.txt} and {@code tokenizer_config.json}). This class fills the WordPiece gap from + * {@code tokenizer.json} itself: the matrix row order is the {@code model.vocab} dictionary in id + * order, and the casing is the {@code normalizer.lowercase} flag. A Model2Vec Unigram model is + * self-contained and loads directly from its JSON vocabulary, scores, and normalizer.

* *

Assembly writes only the missing files and never overwrites an existing one, so a directory * a caller already completed by hand is left intact.

@@ -57,7 +56,7 @@ public final class ModelAssembler { /** The Unigram {@code model.type} a SentencePiece distillation's {@code tokenizer.json} uses. */ private static final String FAMILY_UNIGRAM = "Unigram"; - /** The SentencePiece tokenizer family, reported for an assembled Unigram directory. */ + /** The legacy SentencePiece tokenizer family, when a separate model file is present. */ private static final String FAMILY_SENTENCEPIECE = "SentencePiece"; /** Not instantiable. */ @@ -68,7 +67,8 @@ private ModelAssembler() { * The outcome of assembling a directory: what family it is, the files that were written, and the * stats read back from the loaded model. * - * @param family {@code "WordPiece"} or {@code "SentencePiece"}. + * @param family {@code "WordPiece"}, {@code "Unigram"}, or + * {@code "SentencePiece"}. * @param dimension The embedding dimension of the loaded model. * @param vocabularySize The number of subword rows in the loaded model's table. * @param termCount The number of term rows after the subword rows; {@code 0} for a @@ -88,8 +88,7 @@ public record Result(String family, int dimension, int vocabularySize, int termC * {@code tokenizer.json}, and {@code config.json}. * @return The assembly result. * @throws IllegalArgumentException Thrown if {@code modelDirectory} is {@code null}, is not a - * directory, is missing a required distillation file, or is a SentencePiece model without - * its {@code .model} file. + * directory, or is missing a required distillation file. * @throws InvalidFormatException Thrown if a model file is malformed, its tokenizer family is * unsupported, or the directory does not load after assembly. * @throws IOException Thrown if reading or writing a file fails. @@ -109,7 +108,7 @@ public static Result assemble(Path modelDirectory) throws IOException { final TokenizerJson tokenizer = readTokenizerJson(tokenizerJson); return switch (tokenizer.modelType()) { case FAMILY_WORDPIECE -> assembleWordpiece(modelDirectory, tokenizer); - case FAMILY_UNIGRAM -> assembleSentencePiece(modelDirectory); + case FAMILY_UNIGRAM -> assembleUnigram(modelDirectory); default -> throw new InvalidFormatException(tokenizerJson + " has a '" + tokenizer.modelType() + "' tokenizer model; only " + FAMILY_WORDPIECE + " and " + FAMILY_UNIGRAM + " (" + FAMILY_SENTENCEPIECE + ") distillations are supported"); @@ -152,26 +151,13 @@ private static Result assembleWordpiece(Path modelDirectory, TokenizerJson token model.termCount(), wroteVocabulary, wroteTokenizerConfig); } - /** - * Assembles a SentencePiece directory: it only needs the trained {@code .model} file to be - * present, which the distillation does not ship, so a missing one is an actionable error. - * - * @param modelDirectory The model directory. - * @return The assembly result. - * @throws IOException Thrown if loading fails to read a file. - */ - private static Result assembleSentencePiece(Path modelDirectory) throws IOException { - if (ModelFileNames.firstRegularFile(modelDirectory, - ModelFileNames.SENTENCEPIECE_MODELS) == null) { - throw new IllegalArgumentException("Model directory " + modelDirectory + " is a " - + FAMILY_SENTENCEPIECE + " model but has no trained model file (one of " - + String.join(", ", ModelFileNames.SENTENCEPIECE_MODELS) + "); copy it from the " - + "teacher model's repository (it is named sentencepiece.bpe.model there) into this " - + "directory"); - } + /** Loads and verifies a self-contained Model2Vec Unigram directory. */ + private static Result assembleUnigram(Path modelDirectory) throws IOException { final StaticEmbeddingModel model = load(modelDirectory); - return new Result(FAMILY_SENTENCEPIECE, model.dimension(), model.vocabularySize(), - model.termCount(), false, false); + final boolean legacySentencePiece = ModelFileNames.firstRegularFile(modelDirectory, + ModelFileNames.SENTENCEPIECE_MODELS) != null; + return new Result(legacySentencePiece ? FAMILY_SENTENCEPIECE : FAMILY_UNIGRAM, + model.dimension(), model.vocabularySize(), model.termCount(), false, false); } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java index 657b814fc0..f97152c2ba 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/ModelFileNames.java @@ -24,8 +24,9 @@ * The file names of a static embedding model directory, shared by * {@link StaticEmbeddingModel}'s loader and {@link ModelAssembler}. A WordPiece directory holds * {@link #SAFETENSORS}, {@link #CONFIG}, {@link #VOCABULARY}, and {@link #TOKENIZER_CONFIG}; a - * SentencePiece directory holds {@link #SAFETENSORS}, {@link #CONFIG}, {@link #TOKENIZER_JSON}, - * and one of {@link #SENTENCEPIECE_MODELS}. + * Unigram directory holds {@link #SAFETENSORS}, {@link #CONFIG}, and + * {@link #TOKENIZER_JSON}. Legacy SentencePiece directories may additionally hold one of + * {@link #SENTENCEPIECE_MODELS}. * *

{@link #ONNX_MODEL} and {@link #ONNX_MODEL_DATA} name files of a teacher directory * rather than of a model directory; {@link ModelDistiller} and {@link HuggingFaceModelCache} share diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index c521d5049c..c52d2e827c 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -47,9 +47,10 @@ * Model2Vec release layout for both * tokenizer families: * WordPiece models carry a {@code vocab.txt} whose line number is the matrix row, and - * SentencePiece models carry a Unigram {@code tokenizer.json} whose {@code model.vocab} list - * order is the row order, next to the trained SentencePiece {@code .model} file that performs - * the segmentation. In both cases the {@code model.safetensors} holds one 2-D float matrix, with + * Unigram models carry a {@code tokenizer.json} whose {@code model.vocab} list order is the row + * order and whose normalizer and scores drive segmentation. Legacy SentencePiece layouts with a + * separate trained {@code .model} file remain supported. In both cases the + * {@code model.safetensors} holds one 2-D float matrix, with * an optional per-token {@code weights} tensor. Matrix rows are resolved by piece string, * never by tokenizer id, so the two files may order or offset their ids differently without * corrupting lookups; a piece the matrix does not carry fails loud at load time.

@@ -150,10 +151,11 @@ private StaticEmbeddingModel(float[] embeddings, float[] weights, int dimension, * stripping accents exactly when lower-casing. When both layouts are present, the * {@code vocab.txt} wins.

* - *

A directory with a trained SentencePiece file ({@code sentencepiece.bpe.model}, - * {@code spiece.model}, or {@code tokenizer.model}) next to a Unigram {@code tokenizer.json} - * is a SentencePiece model; the {@code .model} file carries its own text normalizer, so there - * is no casing switch to read.

+ *

A directory with a Unigram {@code tokenizer.json} is a Model2Vec Unigram model. Its + * vocabulary, scores, precompiled normalizer, and supported post-normalization steps are read + * directly from JSON. A legacy directory may instead carry a trained SentencePiece file + * ({@code sentencepiece.bpe.model}, {@code spiece.model}, or {@code tokenizer.model}) next to + * the JSON vocabulary; that file remains supported.

* * @param modelDirectory The model directory. Must not be {@code null} and must be a * directory. @@ -191,15 +193,54 @@ public static StaticEmbeddingModel load(Path modelDirectory) throws IOException termLines, termsFile.toString()); } if (Files.isRegularFile(tokenizerJsonFile)) { - throw new InvalidFormatException("Model directory " + modelDirectory + " has a " - + ModelFileNames.TOKENIZER_JSON + " but no trained SentencePiece file (" - + String.join(", ", ModelFileNames.SENTENCEPIECE_MODELS) + "); copy the .model file " - + "from the model's base tokenizer next to it"); + return loadModel2VecUnigram(tokenizerJsonFile, + requiredFile(modelDirectory, ModelFileNames.SAFETENSORS), + requiredNormalize(requiredFile(modelDirectory, ModelFileNames.CONFIG)), + termLines, termsFile.toString()); } throw new InvalidFormatException("Model directory " + modelDirectory + " has neither a " + ModelFileNames.VOCABULARY + " (WordPiece layout) nor a " - + ModelFileNames.TOKENIZER_JSON - + " with a trained SentencePiece file (SentencePiece layout)"); + + ModelFileNames.TOKENIZER_JSON + " (Unigram layout)"); + } + + /** Loads a self-contained Model2Vec Unigram directory. */ + private static StaticEmbeddingModel loadModel2VecUnigram( + Path tokenizerJsonFile, Path safetensorsFile, Normalization normalization, + List termLines, String termsSourceName) throws IOException { + final EmbeddingVocabulary vocabulary = + EmbeddingVocabulary.fromTokenizerJson(tokenizerJsonFile); + final TermTable terms = TermTable.of(termLines, vocabulary.size(), termsSourceName); + final Model2VecUnigramTokenizer tokenizer; + try { + tokenizer = Model2VecUnigramTokenizer.load(tokenizerJsonFile); + } catch (InvalidFormatException e) { + throw new InvalidFormatException("Unigram model needs either a self-contained " + + "tokenizer.json or a trained SentencePiece .model file: " + e.getMessage(), e); + } + requireVocabularyCoverage(tokenizer, vocabulary, tokenizerJsonFile); + final Matrix matrix = readMatrix(vocabulary, terms.size(), safetensorsFile, + tokenizerJsonFile.toString()); + final IntPredicate skipPieceId = + id -> tokenizer.isUnknown(id) || tokenizer.isControl(id); + return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), + vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, + rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size() + terms.size()), + specialRows(vocabulary, SENTENCEPIECE_SPECIAL_TOKENS, + vocabulary.size() + terms.size()), + terms); + } + + /** Verifies the self-contained tokenizer and matrix vocabulary agree. */ + private static void requireVocabularyCoverage( + Model2VecUnigramTokenizer tokenizer, EmbeddingVocabulary vocabulary, + Path tokenizerJsonFile) throws InvalidFormatException { + for (int id = 0; id < tokenizer.vocabularySize(); id++) { + if (!tokenizer.isUnknown(id) && !tokenizer.isControl(id) + && vocabulary.id(tokenizer.idToPiece(id)) < 0) { + throw new InvalidFormatException(tokenizerJsonFile + " defines tokenizer piece '" + + tokenizer.idToPiece(id) + "' without a matrix row"); + } + } } /** diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java index d819adc6d2..56b6d47486 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/cmdline/AssembleModelTool.java @@ -30,10 +30,9 @@ * *

A Model2Vec distillation writes {@code model.safetensors}, {@code tokenizer.json}, and * {@code config.json}. For a WordPiece model this tool derives the missing {@code vocab.txt} and - * {@code tokenizer_config.json} from {@code tokenizer.json}. For a SentencePiece model it checks - * that the trained {@code .model} file, which comes from the teacher, is present, and it names the - * fix if it is not. Either way it loads the assembled directory and prints its family, dimension, - * and vocabulary size, so a run that prints a summary is a directory that works.

+ * {@code tokenizer_config.json} from {@code tokenizer.json}. A Model2Vec Unigram model is already + * self-contained. The tool loads the assembled directory and prints its family, dimension, and + * vocabulary size, so a run that prints a summary is a directory that works.

*/ public class AssembleModelTool extends BasicCmdLineTool { diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/Model2VecUnigramTokenizerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/Model2VecUnigramTokenizerTest.java new file mode 100644 index 0000000000..5fa5bfc470 --- /dev/null +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/Model2VecUnigramTokenizerTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.embeddings; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.util.InvalidFormatException; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class Model2VecUnigramTokenizerTest { + + @Test + void testReportsMissingVocabularyAsInvalidModelContent(@TempDir Path dir) throws IOException { + final Path tokenizer = dir.resolve("tokenizer.json"); + Files.writeString(tokenizer, + "{\"normalizer\":{\"type\":\"Precompiled\",\"precompiled_charsmap\":\"\"}," + + "\"pre_tokenizer\":{\"type\":\"Metaspace\",\"replacement\":\"▁\"," + + "\"prepend_scheme\":\"always\",\"split\":false}," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":0}} "); + + final InvalidFormatException error = assertThrows(InvalidFormatException.class, + () -> Model2VecUnigramTokenizer.load(tokenizer)); + + assertTrue(error.getMessage().contains("model.vocab"), error.getMessage()); + } +} diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java index 4306307ca1..be809557f9 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/ModelAssemblerTest.java @@ -37,8 +37,8 @@ /** * The assembler completes a distilled directory into a loadable one: it derives the WordPiece * {@code vocab.txt} and {@code tokenizer_config.json} from {@code tokenizer.json}, leaves existing - * files alone, and reports the SentencePiece {@code .model} it cannot fabricate. The CLI tool wraps - * it and turns failures into a {@link TerminateToolException}. + * files alone, and assembles a Model2Vec Unigram tokenizer directly from {@code tokenizer.json}. + * The CLI tool wraps it and turns failures into a {@link TerminateToolException}. */ class ModelAssemblerTest { @@ -122,18 +122,34 @@ void testRejectsAMissingDistillationFile(@TempDir Path dir) throws IOException { } @Test - void testReportsTheMissingSentencePieceModelFile(@TempDir Path dir) throws IOException { - // A Unigram distillation without its trained .model file: the assembler cannot fabricate it. + void testLoadsAModel2VecUnigramTokenizerWithoutASeparateModelFile(@TempDir Path dir) + throws IOException { Files.writeString(dir.resolve("tokenizer.json"), - "{\"model\":{\"type\":\"Unigram\",\"vocab\":[[\"\",0.0],[\"a\",-1.0]]}}"); - Files.writeString(dir.resolve("config.json"), "{\"normalize\":true}"); + "{\"normalizer\":{\"type\":\"Sequence\",\"normalizers\":[" + + "{\"type\":\"Precompiled\",\"precompiled_charsmap\":\"\"}," + + "{\"type\":\"Replace\",\"pattern\":{\"String\":\".\"}," + + "\"content\":\" . \"}," + + "{\"type\":\"Replace\",\"pattern\":{\"Regex\":\"\\\\s+\"}," + + "\"content\":\" \"}," + + "{\"type\":\"Strip\",\"strip_left\":true,\"strip_right\":true}]}," + + "\"pre_tokenizer\":{\"type\":\"Metaspace\",\"replacement\":\"▁\"," + + "\"prepend_scheme\":\"always\",\"split\":false}," + + "\"model\":{\"type\":\"Unigram\",\"unk_id\":1," + + "\"byte_fallback\":false,\"vocab\":[" + + "[\"[PAD]\",-10.0],[\"[UNK]\",-10.0],[\"▁hello\",-1.0]," + + "[\"▁world\",-1.0],[\"▁\",-2.0],[\".\",-1.0]]}}"); + Files.writeString(dir.resolve("config.json"), "{\"normalize\":false}"); SafetensorsTestFiles.write(dir.resolve("model.safetensors"), - SafetensorsTestFiles.matrix("embeddings", new float[][] {{0f, 0f}, {1f, 1f}})); + SafetensorsTestFiles.matrix("embeddings", new float[][] { + {0f}, {0f}, {2f}, {4f}, {8f}, {16f} + })); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> ModelAssembler.assemble(dir)); - assertTrue(e.getMessage().contains("sentencepiece.bpe.model"), e.getMessage()); - assertTrue(e.getMessage().contains("copy it from the teacher"), e.getMessage()); + final ModelAssembler.Result result = ModelAssembler.assemble(dir); + final StaticEmbeddingModel model = StaticEmbeddingModel.load(dir); + + assertEquals("Unigram", result.family()); + assertEquals(6, result.vocabularySize()); + assertEquals(7.5f, model.embed("hello world.")[0], 1e-6f); } @Test diff --git a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java index 298e1a771b..5aaec54d22 100644 --- a/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java +++ b/opennlp-extensions/opennlp-embeddings/src/test/java/opennlp/embeddings/StaticEmbeddingModelSentencePieceTest.java @@ -269,13 +269,15 @@ void testLoadRejectsARowCountMismatch(@TempDir Path dir) throws IOException { } @Test - void testDirectoryLoadNamesTheMissingSentencePieceModel(@TempDir Path dir) throws IOException { + void testDirectoryLoadExplainsAnIncompleteLegacyTokenizer(@TempDir Path dir) + throws IOException { writeModelDirectory(dir, true); Files.delete(dir.resolve("sentencepiece.bpe.model")); final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> StaticEmbeddingModel.load(dir)); - assertTrue(e.getMessage().contains("copy the .model"), e.getMessage()); + assertTrue(e.getMessage().contains("self-contained tokenizer.json"), e.getMessage()); + assertTrue(e.getMessage().contains("trained SentencePiece .model"), e.getMessage()); } @Test From 05398bb677cbc34a1b254d6e653a0ff5e92d5f81 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 21 Aug 2026 21:11:36 -0400 Subject: [PATCH 82/82] OPENNLP-1895: adapt Unigram loading to embedding tables Red evidence: after merging the current #1152 tip, opennlp-embeddings failed compilation because StaticEmbeddingModel still called the removed raw-array constructor and rowNorms helper. --- .../java/opennlp/embeddings/StaticEmbeddingModel.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java index 55133d6d65..6fb031d96a 100644 --- a/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java +++ b/opennlp-extensions/opennlp-embeddings/src/main/java/opennlp/embeddings/StaticEmbeddingModel.java @@ -272,13 +272,16 @@ private static StaticEmbeddingModel loadModel2VecUnigram( requireVocabularyCoverage(tokenizer, vocabulary, tokenizerJsonFile); final Matrix matrix = readMatrix(vocabulary, terms.size(), safetensorsFile, tokenizerJsonFile.toString()); + final TableAndWeights tableAndWeights = new TableAndWeights( + new FloatEmbeddingTable(matrix.embeddings(), matrix.dimension(), + vocabulary.size() + terms.size()), + matrix.weights()); final IntPredicate skipPieceId = id -> tokenizer.isUnknown(id) || tokenizer.isControl(id); - return new StaticEmbeddingModel(matrix.embeddings(), matrix.weights(), matrix.dimension(), + return new StaticEmbeddingModel(tableAndWeights.table(), tableAndWeights.weights(), vocabulary, tokenizer, skipPieceId, normalization == Normalization.L2, - rowNorms(matrix.embeddings(), matrix.dimension(), vocabulary.size() + terms.size()), specialRows(vocabulary, SENTENCEPIECE_SPECIAL_TOKENS, - vocabulary.size() + terms.size()), + tableAndWeights.table().rowCount()), terms); }

lG&(QuwHg1ZNZ%dT?S*5gp zqSdr*otjrTtQPLAR%oLiqkSrx_9ElJ;SQ^7RwmD%1graEw0dSVS-oeXd7nh{KAFk< zFUoI*o$p^ITCoeF6i2%8j4?Y-$8%riD{)&C@2go#-W|p9+pW}-@m5+yqUAggZ>7iB ztqf%5$!KMfX0|6zT0 z@m5QC7VZB;tL;FP+GConjzdxEq^!G6Gw!`G-s(PUw|dSbTD|9@)Yl$GdmBB-+3{A^ zg+wcJ9=aJ@)+#$9PkGzcDt8vT8D|r%`~|UA(NT_JS-e$(EDooyAdCK9&l*)A%P(Zf z&UUsfu~uA=GDHv5u{GX`=eQEvTdV}alJ~7u(w?>Sog`W*=o>iqQHFczD~MUE^c`!} zynT(D#^S8jJ!@#guHjj^mhsDS+HQ$f$E=%~+mfd)=y9)Aw>#GA+sHAXS1}NbxB9nb zGv>WUmAlrcdV)G6+q-wKQ4Q2BTBBOt30Qfo2u_z3sb06eG zvAG|(UlW5%Rl=P!{1|Kz<(81q@Gj0H>d8vf21OO(BPi4J0a3$nH^(Lay{ zXO=40y;Q%${%0uJxJ3VrEZVn3mys1)qx2cxO=XNzR;^g7E3tnOTnm=!tH`>TrK*`& zqHD2xvzF*aWPS4z^gx!#zj>(wc}vg(TOuE{oLizu!kbPnQS-?q>e#!KdjC?jEnCWa zcq!w4OISO1iM~mk;Ke2CpNP^AvG<;d(vOg~73+C^tXFKrdc{Hf*7cmj^-6$5NZPYr z$&j*ly}nKS?}8tG0zZSyu=UCcU$5+C>zO0CUb)CT-1*3YL+e#|VLk7e^(w|*a&kTQ z|9X{SFGp7Ntykst^>Xc6ud03PRSh-R-B8OqM0Ja9Vf^zJ+CR6*8*_^qnltI|DU}bm zf8j0q1^aK>Sgz(nrD{2vsn!Ff^naA9{d6YdpQX$#E>+i9se;(Mr%GvGma4b8lyQqv z^?OP+;4VcMV5K_SSE}v8O11k|(vAvOt7oN}BEp$h8m<@FM*_q`F?5HAt7m?=y6h{_ zI|^r=!j-B!yHY7!AMP39s=Bz6`r}G<5Z5)lQvRZq3Tz6eT^cSQbmQN!FnR4BM z9?#wAFEiPy=KinB&6*;$k6x>~dgf;SV7_ME#MlhXgBh>@D#&6rEW*AFipbYS#I+W?=x=Jp?_Y$8{mq1Z=eSRZ^K0t4QMfw^|26LBkLbt8eJ^Cj{GRsTEj$Mz zlyl&Wa?3uXeiA|bBm$k-zpD^?(ZUE7Us$V>4X>j6fey$%bU${Z2fq!yci$Y^8R)?` z|54SmSz9ogH3m1J2al`^M{j4r9C_w{C~y1U)quV6;=kl8`#ZWH|DykQj+&-^%laR4 z)G}|5T9Ivw=AaWhM;#~srOrjaR~P97Nv|8(L;5`&!w<;!3-GeZ|CXzi^$_cKZfE^Y zWG?Q!8MI&bv3@5MZf5;XDBjBYoevGFl<=|xAE}(R6Dq(A{M#XR^XC<}iM2qGj;sSp zz@E5=wLTbZOP^Zi5YiH_Hl zgiJoo*#7}^0HzrGpLmt|FIO^;jP<|KMa!J=sGXHNT{Xa8RNtlaxRp(f{;0$XRTw*N)WG(JGWIZ~D9_B`P znG4adg888_%nt=W_P|4LsOcf*chV`{f^6me+qReaol}2Q2lmdL%n!ZrN9O-AKa}`A z(95=c(2sopY*RC7XU?SEF;j66Prb+i3D^@MiLhiyK_@R29lJE7GaTIq_Z!Lx(n? zRqq=_rwbhjC%O;FI>y`UcO8%iyBEKPt%GVLo)7$N6M&|Rud4aN8)`X^4g|E(o-=Dn zbU-I`H8VyJ-OvNQS@eHEKMcSD>e>IreUJu&+#3UMfO|X_cReuF$M>@t1B6j{6W)g3 z!$&Za_UBx<23Ellw!H!Qbyx@Af(PL!{@+L54|l@|`*{-iEW8XqhBxp_Lmnq?7xEwQ z4jjXO5)MNZZG&34jQ{VTm1ooX>#09PqPGjz!e7|_1aWW1ej_{rm+*T6c^fjFU0fnTf8c~lkH%b^TPA#THB^aGja4>m|VxmXF{ zfOtr~xL7IYnZu1t+DaeLhWC`tJbvdCbLp82-|9wh1)B3Xm(a=FgpMHRuzgBx?VPjI zoIB2UkaN|A9KhXwk#l&SbBU~G9e}F5_v8v+%)Nvj#In~I6K9@3{xz%*&~S!p2RsXS z7lF~y@PqFHdNl9~We^7z&-7H}Zz;C|26+PcPxvPGtLXc<8g9paH{68%R`?5P-H!Yl@-gJ&@C?`$t5C_a3|xR!d$u3sgp|E%=|w{XHG!|WR9Vy1KCro{{y+6lgvFu z4``V6KQ6KU2eJrvF|y<&>wh52aF-)1`dI&G6YKwMe~tDZ>mNW3xS?4+DWus?$R`>+E(A>I|&Q*PLQ1TVl|_-r&fY4ByZitwx9m-t_A z{IJJkPlj|T!S8GM)gc=o2fsoX#J(8+GVI5&pMWK}t8o7f`*qkq%f0>y_N$PM_+N*- z9{<(m836adO}IM<>w(qSUqwEF|1k1-(C_QOzsJ&*T9E)?uTXT5BScNseE5wA0r zd4e$wbk*JHtV0F!|H?h+vadLy!i}s0w28TW;F!g_KaeHgS!q{%l0n9D{)sNtJpsDomkv~u<3WE2mgB94M;EHZ&MBx zl*9WUGLA)gbz<*;4|lVCLsSt*gb^Z2miu8 z1OErG*AW(gRv3UD7=b}}0}jJ6c!aqB#XIlU*nbPQ+1&pSyO91Ph{x`L1n#LsNWz{B zDTJlYUPS$k`+pwyKQd!K_dhZVcQ!I-5%)hb4|hJYU<3F6wqL7g2KPUdKq-_#IaF+% zQspJ?f8tiLO*OKHG~CEq+;zx$((oX?xEqj-g!_6h3_<9|-s2lm?*isOkw!mqfNlDYvL-wX>={=7-eCo?cR>$yLwg=;%0nk~Kr=K! zE3|-Fi@p&8;0F)XLj!ohy?{C?)Ilw{pc1N~3d*LgqE1L%`slFAkwv%*ktJi)xsiEA z)XB4`i=SmpePq_zVP&2kR!%Z?_V!QYJj0xP$bfW69%jxyq(TZf&M>Ec_MerAOaMFc z0?b+oaTl5MPd)!HTmy5shOXdx{wzGs^>H;ahiBav@LvE=;P({VivL$}7a&WZ6$W4_ zen0t;>kRt{Ov2A`{~Ec1up8m}8165)6;|WE9r--`9DWI}Kq_RwI>Jkk&#=#Gv2DZd;s}N{GLEQ53d+E#Nqb?!fm`O z>=1jF@jr;a$ox-8z@7+6ge5}?b?wxLSbt`V82?1q6y3)RWac5(pE-&i>`8PQk-50@ z%FZkQBI~<2&h!35-xj}OWC?Lf2`i)086XcmkVrmKVF3G1@{@u6Y2@?pGUVX?gs>~n zNt**#!!Pi^k$d3;|iy z>%g2kjlvilgg4+#;BYh!%-hltn1Hu{xo>(KCVwEvMwn$@JZHy>mL>ps`G1eMmoiSoCOteU|m5FL% znusQziDqJ%NE_icxE=0*O>igN1$7xQIpBxM?sUfb8#l+b7`i0&gGH*3a!-jxNtmGs#4zCzp@4``Ktw1YQW1q1^Jiip80Cgc?p^5PH`fng2|L1se-ZA>zQ$^J3f zB_?~pBw7Z zAw-4>Au>vlp#q2u6+mRPB0~osxkizpLylal$k2gCu2W>_@FHUr89Jy)GDd2cY!%wS z$zEtLtF`(DQQoUp@i#%MBlw3bky=e?lvYQJw%+QsTD?xIW3-xN@C+qpOgux;O_HIU zMk(qBalYuhMrR@_QcjQ&-ayUe7HTaU%zpz_ms@pfm~OpQ-_k~Hd`KIAla#kx zx9PS$x?Q*5q1&IoQMcpw%u~8UcYIrSd{1}$Om__TLBBTXPTjfjQQf7xZqr?N|4et? zze9KJ)7`rJe%<}B?tV&mZq{b=hi_~1L)!eXHb1Iw>mJ>6o9@BfqI-4kw{`D>cj(@y zb?a`(haT9X2eAA^59+~9_v%5?d~k~%(!+XqGyf16p<{nU zkKC?DzNbfis7LkaX8xPCP2baF`o6a7KlQkNpdH$Aq$L%o1WaEC!f`myZJ`~zCY8G6y{Ftd{{eQd`dsk zkIa7;JS!tGck3xVMUp?(k2$s<@6ayoGJkklyPngopJ>-l^|YSRGaK~`$M+1;o_Sc$ zJfdezRycppVD9E039I$Y%X)^Ad{)mst!H=Z+2{1^^H1s7pXyo4f47K5q}>~}`(f>V zM7w{e-A`%vu4lEIKfxy4M0g(OOWIAzeQHMPIX$;g&)uo#9@cY@=((r$+-^PhlAhP| z59z3_xycu9NoqF&tbq#=~nr!rD6>7^I-(#v{Td-W6jWEcMs zyYz~Fs-N!Szf1e{GyPourTu!XO0QMxHMd@?)oYDyE)L^#;doFTn*)EC|^Sb8Yd{cIhWa$rui-ci z$7|T3;RFpQYB)*5sTxkxuv5e78qUygriQaLoTK4f4d-b%U&93&F4S<5hKn^^qTx~v zmua|M!xb8?)UZov91mA(xJJWnq1ig@(Xdy;4H|CLuusE&4F@#bq~T@_w`jOk!)+RF z*Kmi1J2l*;;h=`QHQb}&Uh$~n=6XF{uV?D@O1kWE6px2u;V$+CSBe5EBY9w7F z85+sdNR~#jHIk!|T#e*uBwr&18Y$FBkw%I&QlgPkjg)JoLL-$LacQJVBh?zI)kvL2 z>NVogh*u*G8fnysPa}Sf1T@0F+BDLxkuHq{HPWq-9*y*Bq)#LL8X3^2O`~>=#%eT9 zqwyMbXf#2ii5gAPXtG9AG@7c>G>tknny%3djb>^zOQYEu&CzJCM)Ne9uh9aH7HYIe zqs1C6(P+6wD>TZSlUj||X|!IW9*uf6+Mv-!jrug|*JwbaO&V?1Xp2T$HQK4MOpRq} zEL&qa8q3vKp2qSuR-my$jTLFEL?{7_m20d*W0e|nX{<_P)f%hOm|N(Gj@1cm#8uM!`ps^;6HEXOzW33u%(^$L4IyA-}yEGQmShvP{G}foFejT*wpj`)J zbudl`<8?4e2a|O$MF&%LFii)YI+(7589JD$gIPM5t%G?wSg3q`hPt$n1#xpdYsqrk0XKS1Yxf;*Yc)rF9G+wCjB8?YoyhP)r8ZXm$xyCCrUa4`H z#;Y}6qj9&!Yc*b{@p|#p9rtRyLF0`Y_h}r(|A5AuG~TT77LB)RyiMcn8t>3}r^dT9 z9@KcZ#(Om0%YP%>2@k^~@Dw}?yWu5`_i4Oe;{!Tu(_y;~$LesL4#(@TLx&S|I8ld_ zbU0auQ*=00htqV}sl(|yT&lxmI%3liyN<-^NTQCU>4;ND(siUrN1Ak`Sw~uQq*X`S zbfjHJI&`E{N4j(*s3YAv(xW52I?|^j{W>zB37aPDnuygzoF?Km;m|~aCK5G~q=^(w zq-r8f6HZN}Ya&wYSyG( zld+o2&}612vox8l$sA4QYZ5ilB25-+vP6@mnk>^~xh5+#S*6KpP1b1At;t$V)@jnC zNv|dwG})+0pCR1+kOP4Cw8H+!r%d&Dql)ten-OxxM6-t52L z?7!aZzuqKKZ+2a8qImSKm%m8`!^ARzP$LNUnxrwNBlxBxum>-D2u1c!M-J`JYx3ze z`}2kZryQZp5GE0X*>K7c+5%ycH(BzUEP1DW@oV63wren{HJH>IOll1#YJ-W|V4^md z&56}$(#9};Mi}3z_^%ryRAj^t9Tr9?=Jf7yO~*7_85@a&ju9i|G<)P|O{zwi7}Iua z9EIQP+HbZB6+@^HLPZeJyG{J*FEY7rez$chX(}E|$m2ws$&$<=wemM{!H46|K_*>y+A7K&lQxjj40 zR-MKh<5T;mu22jUx6|0V_?v(rf3rJ`kaIe-xTjsA%wad%8NrspX*;%V{$|^56Bpxu z1aXba#4vHYu}#^ycQMjK##ELO86jiZo@wl++Yu2mvNdJrPx2T(WB5Y``;VH?EPoR-b}F9IMGzBX+RNn4xIz)lCb7mA$N$|rW3Zbz6En{2CXP!LgDo)rN0{hl z!#J}c;qg2NF{+K>HU{a$n{+Tf6=qXhp$$yp@u6!lBuL1(Oj2f#7@^3Y8llM3nKci- z&~;@_j7+%0?8)2&#xOCBAPvNHaA=32cc@5bTG?ifDVwl|`%qB(9C+kTop z&pGTIZ^E=M>ETE;`%E;^F+R0VyVx|*>@<;Q4DXeH-(Wm>+GLVQHUUIPF`Hpb+sS*% zblInvJf)aCrI5p16CBL4qlv6in)DNJ+9~P*dHS>d{ocrUrD}rsHus z9ucp_ORQgtFtyd}B9Pvq!SP2Q=RUPS)1D3}(d)50|^s-}g9 zXbscWz_e(_r%k77U8ibEqqb^V1VW-NRj2CGbgDl8R6T+91=FH*S~N_HmTA!y5{<4? zjl^uM3JG75PWec~M;iWIoeHD??f~v)!dh*R2*yrWD`D-(b|Zxbe`&J z(W&l+Q{6?=!lhF^&QslZ_OeZ%?G)>!oa&1`)!&Ion-*EqB7a(xOpE4IeQeZ+;NTJc zWTihdw0VDaC`y0nv~W#}+G)`+En5D)P0RGAt^f9DH6EvJwK^SFDth_B4zUmi@!)_2 zNQ5Lvh7?GJG;l&XWI!flK{n(<9u$K~y9~;q5|{=bUj@}r2OemEM(}|j0?+~-I_)4| z4)*V0e~tnugd!j<2WdG-%RyR>8gK*0;~-r}J&?A;3#9KLeFy0~NZ&#FjwWaZ_Ty-U zHfY!B1djXvQgt8LZDd*0W--T}P>Pc1K<~ZxHt9V%fu2N_MEA}eL~kmy{_b9@-b#ul z5IaBvMBES6%iYV{%hSu#%hPNBd);2I?e*GLuWj|(Mz3x3+D5N!^x8(RZS>jSK40Hw zpZgr2zDdiz_u210$D(i5@^yW_?#S^y^0`Nj>yght^7%(@^XS2P>ip-J|JeO<>^`}4 z-(1dEwsq;aUfR|r^OeuNw$Io0_txj%`utm;d+Td%d2W5|*6rNduUofu>wdg-d$;b# zTeo@Ze!O+Nx9-PV_v7u?cI?Bg{k~-gVt+`4aX7cKj7Yd>xspIfe5 z_sgy0bnAY(b-&!YUvBN=t^4KH{c^i!xnFJ_$6NQytz&uXSl+r{Zrv}pj_IvqpF93L{yY9V zelMfX-|?>X-|?>X-|?>X-|?>X-|?>QehA2=U4A2=U4A2=U4A2=U4A2=U4A2=U4A2=U4 zA2=U4A2=U4A2=U4A2=U4A2=U4A2=V}&kyeB2lw-X`}x8B{NR3m;D6wM;D6wM;D6wM z;D6wM;D6wM;D6wM;D6wM;D6wM;D6wM;D6wM;D6wM;D6wM;D6wMx-xJ>x-xJ>x-xJ>x-xHsgF8a?C-xJ>x-xJ>xpTEfe@#pwI zPfSldPb^OyUw==Z?fbKRf41+>_WjwuKfAA<-B-^H&kWBD&kWBD&kWD*t7rGsGs`o} zGs`o}Gs`o}Gs`o}v-|9s=h=Ps%=GL&d**ufJ^9S`%=XOo%=XOo%=XOo%=XOo%=XOo z%=XOo%=XOo%=XOo%=YZQd**v~-#s%vGkTEnpJ&cz_uDh;GwTcE3*!sp3*QUh3*QUd z3)c(F3(E`3i~WDG|1b9c#s0t8{}=oEVqahE>x=z-v7ayY^TmF?*v}XH`C>m`?Bk35 zd9gn)_UFa^yx5-?`}1O7ULHID{cFscw5F_c|NHkp|G&5X`N!3N|GM(gumAjct7pIS z-(FwtPyK)WssC@U&GarVI5mObKuna+d8u@EjvGOYu$DJ zJNTF7>j!=P;7_aHvW>wb%hwP3`axgcwaxCjWxL%?>%g*)U7y$Wc~e+3K4-Su`R|VH z?DTg2d$|06yOlGydbQs9_18D6$LhBRtU;@54Ozq1h&5`BS>x7(HTAXruU~&nfBn~x z$N%2CANu#L*Y5KF-a7f;N8bh}|9#)v^z8rMS~~joEf?$m_txUSAN|_;j5XW&_08UV zvnSu|g;4F+H+%5S$G**33)YggVy#&l*0!~4?OT6aXV!&vXip{I|JCCd^tc6sMqhJAzj|!B$Fb_M<(`dS-^YJ_fBf~`0sH#L;ji!0|9_lsZO z7rghMkNBKJZ*5vzU+dTR?$>AZ*q^U|_*}=n$Da1=e63$S``+m-``xqmwSGA!J*R(e z%su1ZZnnqm*;C)!zj_Y7)}KwHe7)rs9e|#%TRr@98(*K{3vBrNKRfIn$G^UReX5Tf zeXT$5__|qd*`+V)@z?tGef$4>qJRARzGJ)pXM10_^Y!ju-`D@#ZvQ{nf8J%E|Gm8v zpLX)kEuH-O@_*f~x9rDX|NP_BJAA(Vg>zWGZpgoFh-TP7z7E&_dz=5hMf1w?Z zeSNNBk6%6K)|bUm{^vh`?j;eW&#J zPU*S%b0q$~>u$Z{>$ZIB^gMj6KgaFsuKee%es6s#zrL5At(Ok#*ZSXHe%_=kr&#itzrvH58>r-dG)-QYNkoN5Uvz1xz^JDyd?rXa} zFMq!3YwL49{Lk_6WB7gk&(ZrjmVO|=e#jhN|M~is`r2o2xqI9#UvK$la3_6V{91p$ zOTPZ&e*fC)KW}~Q%hK2Sa~og#xcs&L95vr9f8O!uZvC?Ok@&vy^`S4fZ_j`J@n>r* zzrK$8*Ae%YZ{gE_-m)vcW%#~sXZ36S^R@rJ!*|o4_kH~w{&U}a?U`@5?|#YndHVXn z{PUmx^F8qY{Tci2*UR@!YwK(Mfqm`eKW}|)$ZwSIexrQ%8^!OZuU*)+_N;yD>!#hV z-TwFf^pCIie|@FnJ!OOcyybwotG?d-wL3@u?4bXA9Unhw|7`i|aD1U1nXfH+%a7}K zKa}77DE_(aOaJvV^!@7|`MQ7n*nEGmzHY^j$AqIh;r$cdH|bDK`EFYDTVv73SNx!? z+4?&1x_@u>S>#(wmhJ5MynQ=&=ty{;{p)DH^Xu5LIB{qG?OW>156UI~wQoDmQGR*e z@yp}kUmjQd_V@1hUvt)7=l8eX&hMU?&TmKcxAPOfJs9}izuWmexYqeS^v$~L{2rd~ z{2raQ{CBL|`8|Hv`8{P#FI(%Dw`XT9+nc-U{9btO{9f$q{NC|3yE~oV`zO|Q=l8*R z=l9V?=lAJI=l8|eI<{^*zpvcB=l8!ozc)K+4OoMg+wvU#x98=*JO!_KQe zps)T+zWNz>^^^JPTkZA7f#q|Y-g*7`(;Bv>tSxKX@;Of7ygG$r?9)27ZaS~MKEKy? zdwssgYp=c2){Hf4xs6_5-|O?8qIq>H=C#+?^bJ@;){?bitywr~yXm$!Pb}Nm^!c0at1Y*`<+isRmo59e|-Z( z^~7Cy;@_w4np4|2^|hyV@6m2FJ{?`1%`nUY$&Obt38238Ysia9*99d393e)ybGwCsAIVM0s_h zT3!s7M8W}T3Db>iXG$%R)Z4PHGVfAvKC)syU3PqJS@5pRDjb>6;Rb>4cuSyP=i zkAB`f4tVo7@XZP7x1T?(um28NBi5R=XWe(+di~ooyEnfP-g-~1^Uj-}{x>IG-}94xzja3~@9)}|?xXeGc^k4%LrBA-sVQ$&h5sW-JSEH zx#O?j+i&v}eh%Lj$QQ;fd$Zu*i@lbuEq=Z2mX;hHqqc8LKHtyi+ma)+6mnI&V96eb*Q4d3(AYPyKo>su!r{Ac z+s?JWUHY0!$V;DpY5%Y6`<1)R%lo}ux!o%V%enWr8@J;!(A$mo-`E%D$KRaKesf;; z%{ke(dk5Oz(l_TO-<*$pbMEoYdBr#9=-!-@d-J37=6u_m^Gt8fr@Xx!TF;&LzZR_( z>#p%99` zfA_8a-oIxZShm%FWnEhj)=THznTYoRpEqDz13uT;2t`Ea-PwkBKML>8F1$Od@ZR-r zKLYPVw&M@e`|y3|edLGb>qp%7NY~o7&O7g;_TL|$_c8DHt^YpmbH?qLZ~yo4v(Ed3 z{hjc!iMh`EB;%xyP40Bwr+m$n_f4%@TbBKvVxO|jsSC?7n6lqfj?=XL^|JNv(>^}q zwr3_f?|vNK=UC>JE!&vyvE1f7@4RET;P@;sFS<>Clr$gKuI2rH5Z;%*S@zu@sP|>w zWuNDd)ccChS@rp=w(lGMeckb0cPuxCtntqKrqA2-_NIN?I9dG2_fd*6lsUf8b- z$Krz1&%wLLp6{2w_R>CIGGDs?uN=!O$KcB6UE9vJuf1`<-q;VT>@1x)6`MLP$U+sK2>-OQS+lL2zAI`LW_}>2*wC%xhYuP%pY-8}L^U>Y5b~+zJ z-aq7ahORpw!}fJ}((>;Sx2FgF7`1;MpnZ6N_A$C-?OJZb1GA4YA0PMe@!8JDq}!hK zzDe7kw2xCWosa3Q&d2Pq<(POd^)YArbGAFjJ@5VVW0q~s+otdBkNLyShhOd=3*KJb z=zJ`VSp3W0zs#~?8!NW8y59L%a}4~F{_p_nW8J>3JGSfgd$Y$Hwd~X8isf@R9g9t0 zx8?S>eciVG+IEbcE&SMVe0J>Xu6^6}^?SB=U|R={`GNiPz4&qHV~0NH(7qqq)}gOG zvYjLQbnN(hwFIV=zI>l&C$Eg=h$@TbNZ(9IrEp*wT>rkgdD~fVI}7%G z!FCsX?t=F(*w*5AYs2!f#XZZ%7kzxmzn9pS`z`zB5B#S;@Sn>*cbR+DZTP+Sxw>Y# zeZTuYS6@1xzK=fFep)`hX1~{b?%G4=(*yQTf8ReheT@g|pPRO`d1~3WE${c}`_uQ$ z=Z@pDvuho7K6iU8$8h(#^SSSsc?|G*Xq$(=-gWWMqixGJj_juij?d!-%fHVYm$TW< zr@!5w7mojx?Oi*rH}{=S*QP&RL;dvd_R}@Y&wJZ)t@88UKDu`J>AK*j>w=&DCV#r- z_W9uJJ+%FFo$b?gv`>GdKRvwte6+8wkA3=H`E+gS)3vBi*PT9H!})X#=hO9=PuF2S zU59C&^t6S6PFvdTwB^%ITN&xJqnl1Up0IW-@BjNR>zmbMeYbvCKdoM?&+4~qGRzYs=cU z?BCyC`?qK9TL;#mbz~h|Cl=G+e_LnPxpiS(T36P!#p;<&`}@wiw;rrV>&beyUOLTv z*3S6P_|N#y_|N$L{I@fHzXRGC{~7-o{~7-ozsGIujQ@=PjQ@=PjQ@<^?~HcF?|Y%0 z@t^UZ@t^UZ@t^UZ@t^Vg``6C+J?m;`{GNF=&%D|hzsH5`jQ@=PjQ@=PjQ@<^GqZNa zf5z__T07(SceI`HpYfmZpYi+M)z0|O`2CG-{_g6+PYS)a~wJZKB{wscGLfRF-m!oV~{8#)}{LYiMEB-6~ zEB-5fk5$_h{}ulg{}ulgzw4Xrir?ed=CMP&;=khe_oMm0GiaXcHP7{$GcN6l|BBz4 zmv+VPaYno1zv92*zv92*zvB1!qj_%FuK2I`&1f}eYuXk675^3gHUBmLHUBmLHNWSQ z?VA6Z|C;}r|C;}r|C;}r|C;}r|C-;-SG(rF=D+5@=65c;UGrb_U-Mt{U-Mt{U-Mt{ zU-Nt1-mdwt`LFq}`LFq}`LFq}`LFq}`LFq}`LFq}`LFq}`LFq%fodLWHGgm0HUBmL zHUBmLHNUe{?V8_Xv3AYxxk9_qTTS{@Za!z4A*Y>Z}@Nc zZ}@NcZ}@NcZ}@NcZ}@NcZ}@NcZ}>egZ_bW2Bbv>KW;3GMZuoEboiS_9n6(@J8-8cb znloqZhX02DhTn7gcEf+ef5U&n@9|{wc(OUe)^7N3_&xV&H~csJH~csJ&IL4&LD~)f z4ZpK-&Dpqi!|z;vb1uJmzSNx0Z)WwHS-oafubI_rX7!rqQO!C2<{W=BtJj?8Z)WwH zS-oafubI_rX7!p`y>`oQRhCUtJlox zHM4rntX?y#*IYwrX7!pghRv*AGppCk>NT@^&2@!lRKvwF>ShGtf;nbm7% z^_puA&2zuzx&A6)oXYBX7!p`y=GRgnbm7%^_p3|W>&A6)oW(;npwSORK zvwF>}UUSW(nbm7%^_p3|W>&A6)oW(;npwSORV*UaiQvwF?h)Mi$%IhEQx&TMA&npwSOR&A6)oW(; znrk=BtX?y#*F4T`X7!p`y>`!k&+i;myXU{>_ZYaj&eP2BH8Xtep5HZ}=5cXz3b>i! zYi9VG8NPPU?|D;meXN<`Ypx45Gkncy;%0`gnc-_r7dJC}%?w|2+PInFYp$g=Gknc? zuja9KGsD-+@HI1h%?w|2U8$MjYi9VG$KlNkUo*qk%OzGjB6nc-_@_?pwz%?w|2uC1BjYi9VG8NOzQ zubJU%X876zzZt&f`c^Z;*Ua!WGkooV-waubJ^{ zX8f9Kn$3(~Gvn7D`5*Z`muY7GnsblMxyNSKubK60o)e(jOptY7mSxtaBAX8oF3 zzh>62ne}UC{hC?7X4bEn^=oGRnpwZ*Tu(FW*F3LoX8oF3zvekpGwavP`Zec)npwYQ z)~}iMYi9kL=TptBUo-31p7@{mpZLxEHP5q}nZIV{ubKI4PyA2(E;TgglbYvX%_WCs z_OF@!Yft=U|JoD3`M>tWZw9b=dGBTpus!jc1#B)}HkT=y=l#trU^5HY%mOyEfXysm zGYi;UzG!9vn@bqYEMPMW*qoznPyA*9+Y`U@Sb}LU^6e+%nLT>$C^tg&83s(xn}cR zvzZrc<^`L1!De2tJ@cCvY|f)K^McJKmS$eCnHOx%t2O5}n&+p@ykK*Fqj{d%zMiL= z7i{JQn|Z-zUa*-LY~}@%=;>{&CjZ?yP(3!FsfwtY?e=8~-={ zZ~Wi*y`oRP@jK6(Z~Wi*zwtZo7UOp@;g@gxru_1a-=tr@@tgL`H+~a;`NnVRFW>l0 z{^c9L>A!sAHvyP${H6f&jo-QCeB(C_m~Z^w_)P`o8^6iGeB*b%FW>lGcgZ*YZ~Wi* zor8{Z&@nX_=L2JMFeV4%a%D^p#`T$e<2OMV0}%Pf@0?*=vx)Q8ao##T{4Q;#hrfrv zhrfrvhrfrvhu^vFm^F;^+v(x&;qT#h?lC?5&OfGyzlYy>$n@}=Q%Mhh55Ma}F`t+o z{vQ4wezS_{;qT$^;qT$^;WvU9vxe#6@8R#^H$)gigfTFf9{wKw9{wJFV|VG{cg{U# z>0)3m2IgWWE@t9l!YbyiVhk?E;9}%0X0l@BEk@q*o!_8azVm1X^UaDeCPkp?_6uX^BY`?S+$t%iixy*=Qn8V#q0moMN~u-}y~6#T-+d?~OU8 zm}AO!{_p(X`OP!MJX6dw#XM8YGsQepoEwgrrhMnuTt#yg%~f2ph~_Fk_%&D2T*W2s zXs+^uUvm}BRWw)8Tt#yg%~hOdj^-*q_;pv&UF8S=5B?wgKlpW6aV|PqtZ1>K#mWzU zJyu*V%nyD&R(|mR;Qzs|&B_n{AN+=OqS1;*D;lk6v|{6G0!=8tpf zaj8E)`G4~NxE_O0 zF}NO`S`0+RKvWDwrI+6|hZty&f%do-oL+t{TYC9>`893n&_zQR!&~X&@8fsPBnI;1dUjkpiEG+%{Um+-8oTuI_wo1f>+RzD zUo>~=#m_GhK{yu&qC~+M!W(i{uEv`qVpTD2q zP+E+qL}-wH{(gSfD$~#3&)?7QdVT~55g??Wzn{OKzn|YgTU^&nKYu^J>znE4H{_On ze%CtF&)?7A&)?54Vn{!~kRfIpV@@Cj;$l`HW(CsE-_LJUCjI>V{Qdm>{Qdl{W5ql{ z%oC)a-!-lD^Bbs(`GWNG_w)Djn>EM)zXBoy`~&<-hz#%>yo))64DhQWGQe*nCjq_AK*7r5d}vE`3L!x98q#)kbjV0(Gf$38RT~jI)nU!{Db_1{Db_iOJ|UOkbjWh zj7A3e2l)s22l)s24K8Mo-!<$E@(=P4@(=P4@*8T*Apao$Apao$Apao$Apao0*^eko zqAZCSkPPy>9-cvd*TiEWGK2hs{Db_1{Db_1{Db_1{H~j4kbjWB%irbi@^|^W{9XPo zzro6M`Hfwr%irbi@^|^W{9XPof0w_@-{m)!l`emm-!;^9`MdmG{w{x)zsqk>GhKdV zOq4NE#zYz86+10{*Ywln@A7x~6*E!HL@^V^OuGDC{w{x)zsuj{@A4b|OqajQ-{m(5 z8iSze@|%}Qm*3b}y8J`@W`{GxKg92PZw!paz-Ww>Wr%->e~5pGe~5pGe~5pGe~5pG ze~5pGe~90pX@>X>n`VfAh~Lm@hWLm0%_?Mwe~5pGe~5pG-~2*`_=ot1_=ot1_=ot- z_+*HGh<}KGh<}KGh<}KGh<}LR40MM0%|T>{-=J%T_=ornyk>~ss9c8l4ZmiX-vDfe z`G@%p!p0zMhWUs2hxrY~#x?K^^AGb6^AGb6^BcK~f!Ub3h?$ERoQ)B@4D%235A&PJ zj$zsi^BbtmF#j%GbIjW5C zo2AMKzj>;R@Q?71@Q?71@Q?71@Q?71@Q?71@Q?71@Q?71@Q?71@Q?71@Q?5tPs|Aa z2>%HG2)~)FjPQ@}kMJ8`%n1Jo{|LXCmW=R^@Q?71@SE|<2>%HG2*1JPjPQ@}kMNK1 zkMNK1n}^8={|Nsm|0w?`zd`00lZ-LRjPj528*Gl*nV6l)D8KocjPj52kMfW5kMfW5 zkMfW5o4?N}|0w?`zghf@@{jVHy~!xQd9)am&M5yV|0utq=@`$9!Rd_ho6*TA|0w?` zzhUZ(@{jV5@|)kuD8Ct=jPe_>&M3dve$OcXDE}zG*^P|ykMfW5kMfW5kMfW4kMWQ3 zkMWQ3kMWz8%NW05?U=sRQlWBgGtO@|D&zd){Nw!N z{Nw!N{Nw!N{6=Xr&OgpS&OgpS&OgpS&OgpS&Tr;06KgmDIKgmDIKgmDI zKgnQ~Xo>Q~Xo>Q~Xo>Q~Xo>Q~Xo>Q~Xo>+NhWt z%oM-b*-Y_I@lWwj@lWwj@tdj56#o?e6#o>z8QV z{AO`8#cv)r=5b?WJyZNs{AP1A#XrSA#cxhGQ~Xo>=5;g0Kg~bQKg~bQKg~bQKg~bQ zKg~bQKg~bQKg~bQKg~bQKg~bQKg~bQKg};tiIMqC^H1}e3(hqEH2*Ze7Aw>I)BMx? zx~xp|PxDXnn<36L|1|$J|1|$J|1`fb{7myt^H1|n^H1|n^H1|n^H1|n^PAPnH2*aJ zH2*aJH2*aJH2*ZeS-#Bh&+yOii)u2%Kf`a-KQsI@{4@M!{xZWq!#~46!#~46!*A9( zGyF6BGyF6BGJ(wSO9e8+Kf^!6FV@Kn{|x^O{|x^O{|vvRAT#_k{4@MB{4@MB{4@MB z{N@fb!#~3>KgbOK48Qm%GyF6BBA}Q}%nZLgAv63l{4@MB{4@MB{ImSC{9>WZ^3U>1 z88XX1%RkFM%RkFM%RkFM%RkFM%RkFM%RkFM%RkFM%RkFM%RkFM%RkFM%RkFM%RkFM z%RkFM%WqCRvWd*{&+^am&+^am&+^am&+^am&+^am&+^am&+^am&+^am&+^am&+^am z&+^am&+^am&+^am&+^al&+*Uk&+*Uk&+(h{%^d$6{~Z4u{~Z4u{~Z4u{~W*B_RR5{ z369PqbNqAsbNqAsbNqAsbNqAsW<_HbK6Cu$;iC)59RD2u9RD2u9RD2u9RD2u9RD2u z9RD2u9KU(f%<<3h&+*Uk&+*Uk>l!o1KgU1EKgU1EKgU1EKgU1EZ+1+zuD)Qv&=mIJpVlZJpVlZJpVlZJpVlZJij^V z%=6Fl&-2go&-2go&-2go&-2goo2QOwEoQ1S&u^|e^ZfJt^ZfJt^ZfJt^ZfJt!kWzU z>kBf^Zw@!}{PX8Q4EbyD-jv3M{@GtN$@GtN$ z@GtN$@QZk|z`wx1z`wx1z`wx1z`wx1z`wvR_Q?YO0{;U40{;U40{;U40{;U40{;U4 z0>3%(EbuSzFYqt$>n5_mzrer1zrer1FC@wWzj@=BH;#GZm^aP>{{sI4{{p}6HM)x| z@-Ol)@-Ol)@-Om>k+R6Y$iK+H$iK+H$iK+H$iK+H$iK+H$iK+H$iK+H$iK+H$iK+H z$iK+H$iK+H$iK+H$iK*M7CL63v&g^5zsRoxiP`8Z@-Ol)@-Ol)^6Nvg$S;`6BL5=) zBEM!Ni~Q!Pv&b)`$|CdF$o+3zgz zFYzz&n_G@^FY_<+FY_<+3(&I6zszq=J7vod3jYfK z3jYef?kFq#EBt23v%7l63#0BD*r0~D*r0~D*r0~ zD*r0~D*r0~D!*Jc z{A>Jc{A>Jc{A>Jc{92Bz@vrf(@vrf(@q5jMtnrJqBhtJc{JNv8^RM%-^RM%-^RM%-^RM%-^RM%-^J|o{&ab!2I{!NVI=`kg>-_8d z>-_8d>-_8d>-^&Jtn;t)uk-6tvd*t}$~yl#zwRmP{OkPd{OkPd{OkPd{OkPd{OkPd z{OkPd{OkPd{OkPd{OkPd{OkPd{OkPd{OkPd{OkPd{OkN1{94#-@Ne*M@atl;!N0-3 z!N0-3!N0+;FU$tNZZ;eI8~hvm8~hvm8~hvm8~hvm8~hvm8~hvm8~hvm8~hvm8~hvm z8~hvm8~hvm8~hvm+MR6hZ}4yMZ}4yMZ}4yMZ}4yMZ}1E9v%$Z?zrnx3zrnx3zrnx3 zzrnx3zrnx3zsbMJuQ$sk|0e$?|0e$?|0e$?zn(Ii{G0rn{G0rn{G0rn{G0rn{G0rn z{G0sRwQTZl@^A8Mkh00Y$-l|3N6IF@wk^7(Z1QjNZ}M;QZ}M;QZ}M;QZ}M;QZ}M;Q z>zLvN2eQfUMF+CUzsbMJzsc`K2%?$GCckbjTF~g8vdO>6zsbMFzs2vx3bMt&#lOYx zg$uI9zs0}Bzs0}Bzs0}Bzs0}Buf@w2{}%rizm7Uy>>ykGTl`!6Tl`-9AY1%i03loa zTl`!6Tl`!6UKAl){9F86{9F89Bq18DXtbhX%@+R_{}%ri{}%ri{}%ri{}%ri{}%ri z{}%ri{}#XgHCy}|*lh7{@o({O^J@vC{mwT3HvcyNHvcyNHvcyNHvcyNHvcyNHvcxi zhBjWcA=~`h{9e5w+x)uPZ1ZpPZ}V^SZ}V^S>us~mzsIzAF|D_X^ajx+x*-7 z+x*-7+x*-7+x*-7+x*-7+x*-7+x%WJBHR4i{M-E7{M-E7{M-E7{M-E7{JP0#eY3-_ z_stIf4*w3nS73{#GCTY`{5$+R{2I&b@bBzK@bB>N@bB>N z@bB>N@bB>N@bB>N@bB>N@O$Cr?C@)uv%|m3zss-r%P#*e|1SS7|1SS7|1Q7oIlKJ3 z{JZ?S{JZ?S{JZ?R)9mu^^6&ER^6&ER^6&ER^6&ER^6&ER^6&ER^6&ER^6OT!%fHLN z%fHLN%fHLN%fHLN%fHLN%fHLN%db_;F8?n7F8?n7F8?n7F8?n7F8?n7F8?mSwlTZ> zI@#>;d%c6~@$d2P@$d2P@$d2P@$d2P@$d2P@$d2P@$d2P@$d0_y@c%X@A2>P@A2>P z@A2>P@9}HKv&X;3zsJAFzsJAFzsJAFzsJAF?-ft7$G^wF$G^wF$G^w#HI=f*zsJAF zzsJAFzsJAFzsJAFzsJAFzsJAF@3pzJ$G^wF$FE<{KL0-dKL0-dKL0-dKL0-dKL0-d zKL0-dKL0-dKL0-dKL0-dKL0-dKL0-dKL0-dKL0-dKL0-dKL0-dKL0-dKL0-dKL0-d zKEGFa$v*!+|31GCHT(Sg{QLa-{QLa-{QLa-{QLYG{Ot4Z^Y8QT^Y8QP^0Uvs&%e*V z&%e)qz<lyT^B?njU4b0)Ys7QRf6RZ( zf6RZ(f6RZ(@3mxd%zw;(%&*UmK0C+!$Nb0q$Nb0q$Nb0qn)4j;)&Pd$qD}n z{|WyIztMbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*r+|1WIu>;I$wkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv z=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ z`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp z{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt z|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ z|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU z|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZo zKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>Mbt zkN!XU|LFgt|BwDZ`v2(vqyLZoKl=aZ|D*qp{y+Nv=>MbtkN!XU|LFgt|BwDZ`v2(v zqyI1cf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|F0f?{eS8IOaEW`|I+_g z55NAu^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzrOS9|4aX0`v21Zm;S%L^Xvah|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW` z|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y% z|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5 z|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L z{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0 z`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2 z>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9 zrT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Z zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>Hkar zU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Je zf9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%> z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|7(zckbjVWkbjVWkbjVWkbjVWkbjVWkbjVWkbjVWkbjVWkbjVW zkbjVWkbjVWkbjVWkbjVWkbjVWkbjVWkYE2_`v21Zm;S%>|E2#g{eS8IYmk4Ce~^EW zU;khF{~F{UhgE_yZl{#{eS8ItIMzduP%R=zsuj{@A7x~yZl}LE`OK5%irbi@^|^W z{9XPof0w_@-{tS}clo>gUH&eAm%q#3 z|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j z|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A z|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6jxW!~Dbi!~Dbi!~Dbi!~Dbi!~FXH(*M^m z|1keB|1iJ)zx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g z{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1 z^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5 z(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*Fa3Y%|4aX0`v21Zm;S%>|E2#g{eS8I zOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwKzx4m5|1bT2>HkarU;6*j|Cj#1^#7&* zFa3Y%|4aX0`v21Zm;S%>|E2#g{eS8IOaEW`|I+`L{=fA9rT;Jef9d~A|6lt5(*KwK zzx4m5|1bT2>HkarU;6*j|Cj#1^#7&*um7iwyX%oNLCi9oIkh;NKR|`UD2#HzM?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZKC%B3`}P0n|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(zZdquuwVb5{y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{l7nVe=T7E z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K; z2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L& zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0qENPYx}S5zqTI(5C))Y`>*Z4w*T7xYx}S5zqbF{{%iZM?Z3AF+Wu?%ukF9K z|Jwd*`>*ZC0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiydTsw}`(NAt+I|c`7=SPUVE}q<|7-hS+yC1B*Y>}*|F!+E?SF0mYx`f@|Jwf7 z_P@6Owf(Q{zp?+u{u}#m?7y-9#{L`oZ|uLZ|Hl3s`)};OvH!;Y8~bnUzp?+u{u}#m z?7y-9#{L`oZ|uLZ|Hl3s`)};OvH!;Y8~bnUzp?+u{u}#m?7y-9#{L`oZ|uLZ|Hgg{ zKp22-?7y-9#{L`oF#urz!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=Ui=zqS9? z{#*NR?Z37E*8W@jZ|%Re9|I5upj-QI?Z37E)_x2?7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uwFTl;VAzqS9?{#*NR?Z37E*8W@jZ|%Re|JMFn`)}>Pwg1-sTl;VA zzqS9?{#*NR?Z37E*8W@jZ|%Re|JMFn`)}>Pwg1+B3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&fIt6RZ5<6D8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$=+Eq@0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180G)52G=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=P0*KMi0sfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(E#=@?56>Y1~3}HXaJ)Dj0P|oz-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1( zqXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}H zXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1( zqXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}H zXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP z8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn z1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(`(68K0HXnn z1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtuzzJg4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZz6kMUY{WO5l07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5i{TqMTI(+(Lr{D7{Lx(>+A6`57jgP~BaPV^&hogg^ zSUCLF`S7pLhu?GXzjue~0(bC{KJxF`(?|YdJblCfIDI5gK7Ay!JAI@_IeqMZBBzgi z`#pW^d(Y`(J>=~V??uB1==*2v+H9sC^b^oehPr%yEX4nFQDvZ2!_ zI)&2wzb-hv^0VI4D?eO3z4AZ4(<}d_ zIlc04*wZVtg@aG+)s6GLI)7@fJW%}f%A4*No>EH_cp>Kt!AIdyVKlFg#)8~GBdHUS1mQJ7h zoy+NSKe#!4?tkN_&;675^!bfj;k930pRWCu_jK*oIHzkrzdv330p97_zXt!-5B|oz zPu{xsv%h%%@i(5||Hj4PgC`Fz-hXzu|LobLZ#+JH>*?Yvd->$);n}zD zoBGb%zwF(=`oZDrhl>Y?^E+XDVFP+bScjf%VSDo^k&!3*Zd=D<3Up{;Hq33?({^fUk4K5yD9_~MX_u=8; zql>5Sy#0^8<5$lcJUx8#@cilFJLgk+$+htOaQW=uqHkXDD~J1+-s(-hOc#fThr6%m zeV1|(uYLIFe8G3_{q9@$e&R2_|MD;X@aBJA|IruU{gc1=-tYX+*B^cH;fMd@Pk--= z?|%P_?|%5jhrj#9cmMp0@BYCTzxm{{1&^{D*Iz{FFC-_~F5cUFiG%eCXx#xjp*_2WP+c-`=|SOJDKDe0+Jn zKOMgH=)&>Mz2@EfPoJLe9gmEB@@u||=d1JalcyitfB)`Q`{?nrN3MiRcZWCcaQ@1? z^Y%aU`F`c@TKMv$Uh(Fr#{k4zhwZ0BM zA0PC52_M&!zdYgH?c$w#KXB9grSo@>yBnPQ#M6tz`%msaaFcs-UU$i3iQr~4&0J$Jdg8_eAq?{0HYPjZ|=!o-WQ)e zyu0AtTb=#j^lxT;`8~i_|M=Fuzw@>8cdSR>xqtb{Illee^?CUlZi(Nzc>KUseZIfA zTA$wcZNxp`-6v1K_1tIoH6Q+5{NFYi-tPU_S7^sRe;^A}t}zj|>v{qfcB-(5YYKmMK1{kgXF{U?_{{`POa<-`2K z`O3E61wC`Vt^K@PmnT2-^LI~v`X9Y{>F@sB`R!j`BmbYv{pc3@Gj|vNf6novx4CKG No#V|%ygdB){{bGZXng the [URL] token 9 ▁ 5 0 0 3 0 6 ▁the 12 6 10 ▁ 5 10 11 [URL] 4 11 16 ▁to 51 16 19 k 66 19 20 e 8 20 21 n 9 21 22 ▁▁the▁[URL]▁token +a b[URL]c 6 ▁a 10 0 1 ▁ 5 1 2 3 2 8 b 45 8 9 [URL] 4 9 14 c 38 14 15 ▁a▁b[URL]c +control tokens inline 20 ▁co 99 0 2 n 9 2 3 tro 128 3 6 l 30 6 7 ▁ 5 7 8 < 0 8 9 s 6 9 10 > 0 10 11 ▁to 51 11 14 k 66 14 15 e 8 15 16 n 9 16 17 s 6 17 18 ▁ 5 18 19 0 22 23 ▁in 35 23 26 l 30 26 27 ine 147 27 30 ▁control▁▁tokens▁▁inline +https://example.com/path?q=1&x=2 29 ▁ 5 0 0 h 32 0 1 t 11 1 2 t 11 2 3 p 27 3 4 s 6 4 5 :// 0 5 8 e 8 8 9 x 108 9 10 a 13 10 11 m 26 11 12 p 27 12 13 le 107 13 15 . 7 15 16 c 38 16 17 o 17 17 18 m 26 18 19 / 0 19 20 p 27 20 21 a 13 21 22 th 36 22 24 ? 170 24 25 q 298 25 26 = 0 26 27 1 109 27 28 & 0 28 29 x 108 29 30 = 0 30 31 2 169 31 32 ▁https://example.com/path?q=1&x=2 +UPPER lower MiXeD case 19 ▁ 5 0 0 U 293 0 1 P 156 1 2 P 156 2 3 E 295 3 4 R 242 4 5 ▁lo 53 5 8 w 78 8 9 er 16 9 11 ▁ 5 11 12 M 288 12 13 i 15 13 14 X 0 14 15 e 8 15 16 D 289 16 17 ▁ 5 17 18 ca 104 18 20 s 6 20 21 e 8 21 22 ▁UPPER▁lower▁MiXeD▁case +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 ▁a 10 0 1 a 13 1 2 a 13 2 3 a 13 3 4 a 13 4 5 a 13 5 6 a 13 6 7 a 13 7 8 a 13 8 9 a 13 9 10 a 13 10 11 a 13 11 12 a 13 12 13 a 13 13 14 a 13 14 15 a 13 15 16 a 13 16 17 a 13 17 18 a 13 18 19 a 13 19 20 a 13 20 21 a 13 21 22 a 13 22 23 a 13 23 24 a 13 24 25 a 13 25 26 a 13 26 27 a 13 27 28 a 13 28 29 a 13 29 30 a 13 30 31 a 13 31 32 a 13 32 33 a 13 33 34 a 13 34 35 a 13 35 36 a 13 36 37 a 13 37 38 a 13 38 39 a 13 39 40 ▁aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Ω≈ç√∫˜µ≤ 2 ▁ 5 0 0 Ω≈ç√∫˜µ≤ 0 0 8 ▁Ω≈ç√∫˜µ≤ +مرحبا بالعالم 4 ▁ 5 0 0 مرحبا 0 0 5 ▁ 5 5 6 بالعالم 0 6 13 ▁مرحبا▁بالعالم + leading and trailing 13 ▁ 5 2 2 le 107 2 4 a 13 4 5 d 33 5 6 ing 20 6 9 ▁a 10 9 11 nd 24 11 13 ▁ 5 13 14 t 11 14 15 ra 152 15 17 i 15 17 18 l 30 18 19 ing 20 19 22 ▁leading▁and▁trailing +\ttab\tstart 9 ▁ 5 0 0 \t 0 0 1 t 11 1 2 a 13 2 3 b 45 3 4 \t 0 4 5 st 46 5 7 ar 50 7 9 t 11 9 10 ▁\ttab\tstart +newline\n\n\nruns 11 ▁ 5 0 0 n 9 0 1 e 8 1 2 w 78 2 3 l 30 3 4 ine 147 4 7 \n\n\n 0 7 10 r 23 10 11 u 14 11 12 n 9 12 13 s 6 13 14 ▁newline\n\n\nruns +mid spaces collapse 13 ▁ 5 0 0 m 26 0 1 i 15 1 2 d 33 2 3 ▁ 5 3 6 space 127 6 11 s 6 11 12 ▁co 99 12 17 ll 105 17 19 a 13 19 20 p 27 20 21 s 6 21 22 e 8 22 23 ▁mid▁spaces▁collapse diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-identity.model new file mode 100644 index 0000000000000000000000000000000000000000..54c3482777324a10fb4293c40cf1a82b9727896e GIT binary patch literal 5287 zcmYjV3v?CL6-~lN7)4x94~kXSRTMvk;Fl#DwFtCjEj}rqV%6Gp^WMC?@y(lg%!ioN zDhVJcO4Vifh#IBQ`auHni=r4zh+tI$EiP^Gvy38Ex@r+?#cI3nJu~ydTD;l&+;h%7 z=iGDe+=+3bF`z1R^4C%s{+(7~oM7~;@*~#I7-ICV8dIV}je%7K>Brjr4Orhad&ZPs zvvq&NF#3JGyph_r^zSeR8Rb58&bmUQO4{|GsT!Gd-MVUx96JB4x(?%bqd#mU^4DSl(a55!vsLux^eKKd|%CY?Eat@e?o$!$O!nVY3!BcIpWGN9wbHm#DS z^wjkoSf%4sDb3pg8hy(K4mD3>RsynhO+EE)0z>LG^s`|vXca%8@4nd>uTKq~^UNl# zQA?d8HpiA3lic^g5T*3!?vY#KfxxBixnLM@L%OFW8L(YSzW|208pz%VG}IV`@Ct6m zv}x=n2+$FMeZcDVmGd}Ix1A$sm)2B6`1FPBU=!Ro}QD!D>eQ0ak zO_meTz2CQM8x#3WJoUv`F?4A5x|i64N9&Jm(-Gz=xc%i=#dPT3_a_YT)gDRM+>qWK z_DY;foea>9T|a_=!!O#hp1RKkSJ4Lbw0kNrhgfIPdvm}LyPcue=ChJ47U_=a?Q!|j zwD@R8Vj(G5)u{ulqn7KtVmbOGHE?H#G0Yf>g8F7{!E^$tOZk`Kke_4>R`U8njcb#Z z#Km68I)X`QsB z)f?Z|(6;H-{eT=v)}v8#_V9p7cLjhb;%#)~DMpu2Ll>jV>2`&XWlwto_s(Yso3rUo zHIBmPq$ks=gR}rcw=&`xXFJSzHPfzF4E?^6g7osS?J33(thnQP1!-Fiu@o!PJHLr- zSfl8?jR>H9W+3blJV^09a^Lq>Rao{0&P|#YI13Lv|}fDw4+V0 z)}-gV+14*gczSRb80HT=mlnO9m}V{8q#O6}n1ofQ!W;!6a5L)6cHe;v^CD%FW_$p` z4TRw7kfQ3`@GO%)Jd%v1-2!bldgAB{AzgD~!ip~S4g*6x4pO=QOkj4I^&~Y_gTkdc zYG~sxnV>4uXj#SHIL8{-&HHp-3IZ%F_U!txfN*HyI2<$u7#?6Wj}HBo2du!P#Scc< zG3nrC`{L;QkcQ;JVD5SW_5U4XAE#SIYb1xovQz%7XG4q&+W?_n#M!iZ>i*bK!K5cv zzpF=Nn5HxKzNfpgns$EuS8XaG>^u-vvrwe(Z+Snq&A7B>2N)Lu$2|I zn9ssSIrAG9XS4MA4=h&wbMfX6wO?}{+Rf+yL3daUd<^|daTKbv725)kdumbvn*a=* zekr<=(RGZJ@`eQVyikIhOYr3q+|H0UN09SOnPJJv#tB%5z|EUZv=cB7cmF&$Eo(ws zdiAV|1bZ7-B>3R+#R)#Vu$?hSJ~M~?0zcgFfmOg#X+e~jO$$rISy`}bT3izNru1{P zS_wE_biR|3^kPW`y@xWsp#*3o+sd2tUqzxFXW?MgoCEL!iN2Y*6AHY11zDt<71hR= ztn+myzE$f`mm=}Gj$>`_ZXPU1oVLgP2_(wPQ-anv6T#fjMHi2ID>3(^lcyW@fb+X2 zDl;$o$ezSdHAOv*-3tMVoWl!$x58Ylx*YA>4;Iy;ShP{igMVY)f^_KC1B@-)I&MD1 zSUQsK{1TWGR(G5=M?sOWdi&9X{XUBQXDwPV7>qOaLdT(}hw-R6H^dHlRxuv6T^u+I zh9?}aW1HSMgSF8|Ig8%D2n<=I5oOh=L^DtqivyQ}BZQ*byVqXHie@^XudW8>bPFzp zRjejGOVYJJiEMEC)2`_t*bnolyW)sO)9Bm*-)VFi-f;iZQ};GLSHexNmhghMKKSJX zfAZ4W1Rs2KLxMkBwJpJ4ENUh%@_)E_NrL~;ypl1Zx&V6s5ddA7peqtIEH5<>|t+X396SIm+Iq3TNB*dSd#ZPuIeL? z9O?sLT}j?l(*LA=c_RO${rM7pB?=t2pFa6YBL8�|`F3;>i+T*;c}jtWEG|&G(k@ zb0wdLo_#ElA9}SE^WQfvOyplIUD5}a@;&_ULy7$GV~Y~}&-ULlMyGz+vN)22IjH=PR%4^yct zLKA7!c`Tbw%@LYFKSbzysvXIrv#BRSlWE-fESp4YA~c?i3s^Rj{0PmZ0}+}x~Z~rx(wr|#;NMC!4*cRr|AhCmkRo+ALA%dGNA@z3&!Y`_khG`~B4Nj3$5pk(#t}sqkA4#y)4<8Ev znmP-13J7+}5Hu1SE#=0QvrvoDT?Fe|MD^4<=+7`Ls6gM~8`i9a6YjCBWqOsVG`!4; zmaC5gNZ}(&$kZzZ5jz8bKm0~r2De!5+9*8bU6lr_mBoKdS82>P1l|XDJ>X4%YY+|< zfd|eX94Rj^$p)Q~=V1WOvaNAUFTr>%F&4(U7P(;QI z10OB!iJ3BJ5GZ6W5V*wVL{6tVOL>~4mQ;bf>WzjdXoQR<@P%qzUb&g7dm6N+j|LYx z?mY=TD8WH4%581AQ8hiA^-Y-L?=Yf(n_1|gUoa28KH%7~FodR5{pxxazDLoL@ux{; z%2v@#7ul?8col@=$`hUilpbP4%3FbBPwNI!aMMRy0@I-Mm?&!gcrw&%dAV`< z#Cm+QSZTGb)%jFV`zC8S86=7jYj6t@MH%E&1}?T|QTl#m>J}B&L{X3JUl1?`RXa~P z5D@Sol(mGD;jW27sWGt-T3N!o0-S^PtbC~ad;~Vf5b94I`p4J5NLfjgYxOj})Md&* SDaB7+EG^;sXc)!9V( the [URL] token 10 3 0 6 ▁ 5 6 7 the▁ 11 7 11 [URL] 4 11 16 ▁ 5 16 17 t 9 17 18 o 10 18 19 k 41 19 20 en 39 20 22 ▁ 5 22 22 ▁the▁[URL]▁token▁ +a b[URL]c 6 a▁ 21 0 2 3 2 8 b 31 8 9 [URL] 4 9 14 c 25 14 15 ▁ 5 15 15 a▁b[URL]c▁ +control tokens inline 21 co 72 0 2 n 13 2 3 tro 119 3 6 l▁ 61 6 8 < 0 8 9 s 7 9 10 > 0 10 11 ▁ 5 11 12 t 9 12 13 o 10 13 14 k 41 14 15 en 39 15 17 s▁ 12 17 19 0 22 23 ▁ 5 23 24 in 36 24 26 l 15 26 27 in 36 27 29 e▁ 18 29 30 control▁▁tokens▁▁inline▁ +https://example.com/path?q=1&x=2 24 h 24 0 1 t 9 1 2 t 9 2 3 p 27 3 4 s 7 4 5 :// 0 5 8 ex 120 8 10 a 14 10 11 mple 194 11 15 . 97 15 16 com 134 16 19 / 0 19 20 pa 56 20 22 t 9 22 23 h 24 23 24 ? 296 24 25 q 298 25 26 = 0 26 27 1 101 27 28 & 0 28 29 x 149 29 30 = 0 30 31 2 155 31 32 ▁ 5 32 32 https://example.com/path?q=1&x=2▁ +UPPER lower MiXeD case 18 U 230 0 1 P 80 1 2 P 80 2 3 E 144 3 4 R 235 4 5 ▁ 5 5 6 lo 59 6 8 w 48 8 9 er▁ 26 9 12 M 154 12 13 i 8 13 14 X 0 14 15 e 20 15 16 D 152 16 17 ▁ 5 17 18 ca 73 18 20 s 7 20 21 e▁ 18 21 22 UPPER▁lower▁MiXeD▁case▁ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 40 a 14 0 1 a 14 1 2 a 14 2 3 a 14 3 4 a 14 4 5 a 14 5 6 a 14 6 7 a 14 7 8 a 14 8 9 a 14 9 10 a 14 10 11 a 14 11 12 a 14 12 13 a 14 13 14 a 14 14 15 a 14 15 16 a 14 16 17 a 14 17 18 a 14 18 19 a 14 19 20 a 14 20 21 a 14 21 22 a 14 22 23 a 14 23 24 a 14 24 25 a 14 25 26 a 14 26 27 a 14 27 28 a 14 28 29 a 14 29 30 a 14 30 31 a 14 31 32 a 14 32 33 a 14 33 34 a 14 34 35 a 14 35 36 a 14 36 37 a 14 37 38 a 14 38 39 a▁ 21 39 40 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa▁ +Ω≈ç√∫˜µ≤ 4 Ω≈ç√∫ 0 0 5 ▁ 5 5 5 ̃μ≤ 0 5 8 ▁ 5 8 8 Ω≈ç√∫▁̃μ≤▁ +مرحبا بالعالم 4 مرحبا 0 0 5 ▁ 5 5 6 بالعالم 0 6 13 ▁ 5 13 13 مرحبا▁بالعالم▁ + leading and trailing 11 l 15 2 3 e 20 3 4 a 14 4 5 d 16 5 6 ing▁ 30 6 10 and▁ 34 10 14 t 9 14 15 ra 76 15 17 i 8 17 18 l 15 18 19 ing▁ 30 19 22 leading▁and▁trailing▁ +\ttab\tstart 6 t 9 1 2 a 14 2 3 b 31 3 4 ▁ 5 4 5 start 187 5 10 ▁ 5 10 10 tab▁start▁ +newline\n\n\nruns 10 n 13 0 1 e 20 1 2 w 48 2 3 l 15 3 4 in 36 4 6 e▁ 18 6 10 r 33 10 11 u 19 11 12 n 13 12 13 s▁ 12 13 14 newline▁runs▁ +mid spaces collapse 12 m 22 0 1 i 8 1 2 d 16 2 3 ▁ 5 3 6 space 117 6 11 s▁ 12 11 15 co 72 15 17 l 15 17 18 la 98 18 20 p 27 20 21 s 7 21 22 e▁ 18 22 23 mid▁spaces▁collapse▁ diff --git a/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.model b/opennlp-extensions/opennlp-subword/src/test/resources/opennlp/subword/sentencepiece/tiny-unigram-suffix.model new file mode 100644 index 0000000000000000000000000000000000000000..984acd070e5b5615344f54f60bffcbf8ac24b044 GIT binary patch literal 245428 zcmZU*3s{urneczl3q6o=FdMKF>Hh`O^g0wwO32j7WMqf ztEFp8Zf1D36zv<$_%RsYgg$PIw#|9K>$zFGIO#*L7O%~?aLAX*b`@=PL50_Uo3`=k!`=$K;;X<(;}LJL z#H}jWk*P<$4->a;Qcm5d-+N=dq0M}NMv)V}6&*d_?DgNS{dO4|%@McNTbH0F--9=` z-O(R;OP9D+`>!8WdPDEjZsz;6P1=9Xukt=l*s7H*tM;nf6SSAt)_6lEX@)SXFWv3l zW(m*7NnN)kXxpdQyFU%dV;qb|3`Vy+$lS}-4$Nq3%qSh+)iz(a!|lG zcB&Lz6}YI=B5m<`w2%>jiCcX|U;J;raKHAp>_0!#>1`?MsMT&jr&94LTH&$BYMl0? zhL63zZzgJUZu`{u&02iPAH2TXuvD zJ5o|M`W!S+{N{}KB=z*azI__EB}p4z;tTp_9K*n)9BP1g8}f~haj5|kw_Vexed4Xk zjyUZPK_|Tpo}98V!P_w*PyCDbSxSbMwcn=_zE4 zz@Pll>$xRP+qwmn9!pI3cFUhfp7B0;bG!DF!ESHx+1lGb{#UQ|ytezXws=Te>C;A! z7_l{Fdve^C#NQ-r_j%D#=URu<^G2Uao%)U0KAn#Em1a^&(nM*rZ^Bj@G)H+*qU5;k zNu-tg*cqR)eOpGFmam-j&>L~S29Uy$m9n<@dbM`aZ7?`TI>s!POyO`=oCHSEp5g!pBlSWbFKA-NK7a6FVU%Q+Ro%R zQnq;Kn;nUGk#=&z_DuZBU9Q3(TyjJ$dN4Tiv*0^Nt&e*eFJw7E_ zJNl|me-j(%EA;6!vi8T?bH4EDiJE)bd0&nx+UTV|b!&=di}=hNB4K-)HYNFUU)|J@ z(}pd&dRZ^X!y?TMQbx5jx*$?HcGr)H$k%U&1l zpT2o}Lfl55lhS2uP299SE?Je%8v?Iir^Ls-k+CIi`%a&i8u!P(Q=Q+boj#RRfBw?T zDv2yrT%2*661HxP^EoNOn{mmBTYN4`ot~1Eu+?i;-rzh>_;muAWvj?zM~q@{zY+IF z;+Dkp$3rSdJ)jgPKEjrK>N%HBWt7~?ZT0Dl(5)&ZRgia7IAt}7@kv^-awt1Bm20Js z^}oDd;L{1TJ!9)utwKE@iS11N-X?8Q9t`Z{j5O~&_^EPJ*!0BYgf#7o$4?XDeB=4Q zzYU$@3q7dZbX@%Axb1QAtR~(xwT1Y4TYWRO7_s(k)t3Ci7iAm9qJ^yS1xt?Gs;yBvX}`fH zMU9c|n|#rH^KNgVFJyXRyjGTkPN6ckdbfaIB;&-mw#04Sl)=zek^f_xFL*qe{37#l z)RYu0&4f-_mV2}Kc|Ick(}zptqo%`igE*LiVzu3E553w?w6nARug`=I zYpFj*CBa)|g+862v$P$`7ws2A+Ort9&u`k0HfKniJEScb(iRSB%Z9X7L)sdjrtJ90 zcWU)_YR!;(?(lbN(~x>zO;NsToL393uNr@@_-IJ)Hy!vp{c$2xwclhN^8ZV@-g}oL$yc&+Xi3NPIPE_kJWRH+MdU?{Hr5(NWZzK@bC1GhV)y%%Nx@Bv)$SUkK^{| z*#A!N9MZpNegE%t+jn|<-FJG|ksrE5t4%Lm`3`Im~uA^+=Td%x2UbPVZVx(^KL z{ibZq@om{m_222)-}zr_C>ip zA=0TA$yEnvG3vncH@&p_i~k7kP}Aaam0gp z6U8_amO?ZbXX6i5geyWuzG{$&Wd>P3LX1(H4Wi4V2s)h@_KE0=MEuJ{{7OUuD?|bg zhzze08Fo};q)lXmMI^XQB&bUy6hhEPpA;E&Tx3kQ$oEc*XgwlNoEI6}EAstIBIB-c zJl<9B5!;(OcI_Am5s&_+4Om+y!lnA zByJ0m&9j0e>8l|5)#pKyyewF@ObC{gQQwoTTY_aJ_1G3PM$)Rs$o5-fB;(mplK#>t z*>P^P{Q8Mevh&>c<*lzr$r|$b&DYu}!e#nD2hG=)<6lss6FagBjF)t@4S9T`m%qNaQD2N--~`hB zW|Q7DI#9Cj=;W)QefpwU+Vz@FLU~rur=x(lgQ@dQku1t&LK@3-;+e0L?k9Az_qa~- z@OM)dv!;_1`^U;34m}|!bH~ajFOQW|x5mo97{4b42f6&sRGF&Eb~{k^ zke=roKN*WVoNxuF1H?f&N{H8q499;bVP+z;c>X)p*Y^V?3~B5M5YzDhZ+mq6`AhaH z>X;EIMjQTZ0g^vAPzt66O70?&a?+^6?>B;f+8k>JF3&{ECkU%3ks^qyLSuRJA@p z8W@u`xb0wG6d=*WSqa7DxfHnuY^edV0r>~Y==p|m@ti?UqMNbtPmz0LL^_CH#W*>R zY>W($0@BSd6R8fPOhqE6@XxNHZ+b**x!wn-;B6*}^ue@QdHMkfX9MG6@QMU-n3X{fQk z7r*RakxPUrUM|vbLu3cf^rUB=#<-b6da*jm!mnyBX&cEu2WcL}O_^*akpbd04mU^l{;*h9GrlS8G5xK9%2UsSv)A{$gc z&|V41&~WA%2#2vS6&?}SIE^|%2w?)l#F)F2wGe5FVeA~$$t2P;Kcsx~=@-`7vFaJ^ z(fzibu@o#bc%DSu>eHc8ut4NLX#1k0p%R3D$!NwS&wBCuJ?>O=V_PV89V%zZJ7-_0 z48t#6<;ysmikwQgnaDXvQw3>GqCK3deaQby@@xuX-VKtl$H;O`)>4C{&c) zE+F1K{FWdaPZO7PSK^L_0{UqUGKV~)d2aTnEcfY?YJ+URe-(avm($O8urJ(2gj<21 z8V5UYr+}x9arUXmRq9lbDAJEKbzy_F+h3Gln@-jcr^MnLPvyi}kGqg^97aD_A`*}L z24R)$e}Vov{2Tly_Q+}#QBF}Ou+a68i9qLg;TNe_xnDVFa%maJL zql!7Za=A`+Q1%AOWM0eKP{Wu6lY+V!!Ir9%Ou|@Iy8CEH@`(mB?Y#!M5;lM-nefo) zual3+b0&US)FXp<)#O)Ljc zeTv<9B49K6i+ah45pff~;1Yc|gS@GuiE)%oUc1QiEYD8CMU^IFvXJ&D?DLgT?J+#L zr1P>)%u5WCyMQ&EaMxAXKhbZ?^sT3P6ww1KMQ0AUL7GoPog@lOFyp`VyC zNE_Z!&zz($toyI%G8CC7y8Wpoltpp4CYjdJ^2F{3`x!q;WlA6Bsu- z$`FUn*fTB{nNGOpXsa&5sJ%%P`irm{|6S0*zUWo*%nnm~6xQ!?fy{->XN;$UY0QTu zfig(>-brLET?&vwWYJ)t{DL^etlfJrF%Od71Hw!uz5k0WIZ8T__5_}H>G!Mt(s+Zi)6V~gbT`7UU>o>>B_U8Q6VKCXkm}e#=|}%H{x^{i zz`z`R7a543*~a+*xpzu{1S=Usx|AzBnKoe`kga2XLOUJAuK?*#ay0t`n@+6go|h>P zdpr~UW;{(>BGvqFd`RE*=%kb|ds#1v=8C*c`f7}+H8p<*?S)@%CTA4NlbuIgbfa2Z zm@mfQ=biuiunFQ7!YCz!X;1pOiZQ>3wD;rpGuqQUMJF#K8`jb<_$xc=K-x5&9L0Y+ z>E+}1Tf&b=pA1#>jWL+~7y|{fsrPdFY@Baxp2PE}iT5GlZL4Va<zw=%jj)$U>fd zYtN?J2H^-{tc1Ok?GfXq4&8JT`^VOgp=YBnCQj~YgLjPNlh_WaUfRut`~(_%ME;0OBi;$(ols+ZmQMbebYFurDlI>q zoFhyUVX7=5e?m9!qfSpSH?`3pP?g6ykMP+E%s+%vdHCb@+?pWWe`B7*uihr|XX5Tb z_k7H{evbWT7wz?KpnQqHiT?LI87AFN>qVWLzQ(V+m9@VvKn@b;zf_oMB35*>;x^%5 zAUt~^2ks6q4QKo#z5eI)Qq3IjkE$Hxq2^Q1>*TvJK>nAs{sGPqwq%-40;spCJ6tZ{ z7W97PWtGk+gr}?~+T3_aWD0Sf1-16%EMd$d)f$w)UMDwct6XG1vYRud=PTx1-2^Gx zsgnoz700s1#OdTNx~X@fn6z*yB@fg3aM^1Lmx4v%)SrIa%-RqWE+$L3Oee3L8lDq& z@A--HL)-;UCfzcg9{}1Zrwex;dQrICBt5myvQg(Ep5Mi-*3?ky^Z?x$On*HTF)$CC zFNKSF=R`5ncN))abHk-7ZlY9?pM&_hgl$8b@t;k;Hqy^crLQN2OJn3jX(Z1oo?oUu z!vuK)|3>Pw8EL_P3`HCXlVLm<+o(TePof>!v*pq*)A7$Q!j>2lh1hV> z4V_#d?L7}^w`HL4RttDOV2nw+!^D52?_`OM5Imy9NRUuMo z!ynlgOgPrZ>pZJ+;t!Wqp3EPno$edW6E6Ec_aEE`s6DWeI|MZr^7{fLm$%F!!MpmwxhqIHmZTmj}uLq=B&Ju0R=y98WR+O7TV_KeAAEKl1k{4DRZKF zIkQm=bswSbfsP!|L`=JTv54_o^pP~1m3TaurSvh1Z%yT1rx>;_2@`-;L?&OMBYcOcKn zz8(t7xC1m{--LewUV;yvW3He-DmkBZ?`7;)2S{EbcenV>22`WxD}w|g=fXGS*H|X9 zleiA#G3Z2h!zuU!*dc?km6Uf1@*w3O`wxVPHOL=%HU>Qu)=i=#jQgzkqxVNuzah7whQ^lsuJ@jx%X+sKk?AJ8|;-MUv20 zgDa1*F^#)`DWS3fclM-E$(tt9L_T@U6`SyLa1K6}7b=aXb<)7yK?`T&X2y;an!uEx zW6&}-8H=szT+L*j%Q!Ihk#BFPq>#oM@O;WRVc_O5C+2Ca8PstOc)lDb-A&`i_!~J?G_6ITFi$lK9#5X)0|L1=q2;6yyN%H^`gFCZ3lDv(}>Dh1{MX$-5CE`{|s5 z-Vm|lcc6)R2X1_!VT{+EB?U5;^*Ou$Y!$yvswRAAbvHpS(EQ;;ScQ?9*aw`-D3?jJlgM zqxU>@ocn$J-S7H)+qv-;19_12@jpmgIT>qHdEP>w8mWVc`goqjo~ehi8~6PXZ$IVG zf5r^@pb>co*@K-Cej_^toF5vol3XdA`4ove90pJR46qb+@GMm?oo_GRF4L z;9h}eD$Ml9=~UsKqjbhbHevJT3i%PQk2^b_mBT{XROPu2nM0Yp>(VsF=U4P=1N#%& zYz@y=!w+bxHKexzjLh98^4f%+qTJ-K?#;3pBkOQ?v-ZX#lOPQW+2bE34e$A^ll?E} zf`2FNeD(q5JU3#)riZk%N|$n)mJ4S-?DtQ!0rD_m4uToFS%3F4uN078H|6vM(N=B5 zC665Dk5UV|jWl{lqXJpOnnBo0J$soH_nykAssB5Xt zkNJbUrmywl-B(hs{j8~HD8pH}0N(RE`);*QRd;A+;;XhcvcFU3wC>G}q5Y&WYB=|) z-xtrTA)NCW7f%E;&yQeyaF4~o$a8fp;GREHy1yRg9bbQVB8;Ltpiy zdp;+fTkOe|zu-@v-Mm-o=1!se71G~K-dpHDo^Qm~_8(ww`FNoiv3JuY#sX#S-blE& z2=_LAHpVjKCD4Zz=&SI9{e8?!?Df^$P{(R4RaZ_e;0 z_6()8P2ODq-a{>@>dr-t|FQV1wDK4S z571}PH%8We6APTGw>lRaNF#48cdb0X3p9$io(0O@>6_de+)<(DUt_&e&*@7we#_PK zFpNe{2R329mX!{ znU%jPKdj{0JoP-8=d`IBD>)XO1gi2=UzPtz^hbml51vnHk3Z>U4s|}I@?Ai=;4Jz@ zI0F}yKXp*`xr`pJ(m#p2hdxo~H1)ysVPxqJox(M0Bvoz>t51NH60M$>$ z$Q>|~d?qWD{}f-DJl{~yPq0C3Zvo{pv925U;r@g+!k+V15pKEKw=iCrTfV(tFeXNc z2^N!HG<4685YH=)Sw>jZhRp3tdA`3bO0FooI!ZjqhQfUjvXWJ5DJd)1Gk$ekx2AX?W-Tqde!C8ppZxpB;bIk1fbT>Yja` zXO{@m4O7r*17)vno?F2I9Z=fH_$Q4M=-%`+@r9p3_(+j7RbDktNKdVQIVSXTkusBd zdY)GH%UsF6eh=ZADQ79Nhz%hA$-L9TuY-6c3;7W89C=Ouq>>SnE`>}n2aW34`jUJO5>PAoab-@yasi@nHS zf~Qa~?;*RdMlseI^VrQjY)`}QQ{YvV(MUbJKjZlqQIdw+(}~_1#lDUG0DFXN^2tvW zIZ60D)_gU--RPb#Y1fEoDWraC{n<&}LfYdGJllw_#&4lYzlS?bWPTquO#T0r=W6`! zS7ETxLi((58sm!j=QH&Tsml2w&+}$*COJZRwoslP%0vCLb?hA-zl#eahM!?4j$DL?&nN`+JY&+5gFRCmDn)XNygxv`u2F(y0_?78NW z7JK7-_IKMZx)hs>1@Ki4Qd^F9_it( zeUp;RhiWd$C!RW|6;s~)+*z^$Y2;oY{~_N8sBo-Bh1g69`f^Zux$<8|`q;>S6Q0+7 z#`2H-<$czavL`th)BmNfawaCvLdx{+HxZIg`~0rjUvig-{8YvNOfMfc5q7=E2X+2Z zb<0l*RbPBgx~!k(PCqfLaIf>t5%MK)fo+(de1f!o?Z^8X%I42~G47TxIkQgi6X&z| zbB=Q$_v|G9Ox|Ut`$;}y&GixIjST$4M9vYX{U!RHvqJ0N7$eoBMLQnDU%i_uP1f`Wj1}X8C$UweN@E7P^K*kVaduW|TzXY6FXN};UV4Qz$bTyCtLfZp zaQ5!hXdC2Jqh2!cHyzec;+l%jw1jb5&P&ApAsAZ6#c zQ^e14)gYC`siF+;vCsS^ey-zuvx6*0ZbqiSuR+<5r!!hyY}ie>JKqYDF7)=VgQVk? zAZbMw;$H_XtwGX&baI(^?1(`eM}njrcjF5Nsb}2VvGG=az86?#kl*2Nf$A4&SK8nl zdKK~`+R#$SSeeUQ8EKIBD5n#7U_EF37|#6I*oTSS0U!^-;XIxEjfh4KMe=Qe1ChMHK^|o6d^ntW z5B)S;U8Yyx&`I5Eda==tU*UHPzJ~vVg4Ydl4QZ#pT-ZlB?fwn!t95$$8#4bJ&Q-Sz zae&Yy7diC$pClQ3Fe+9pC^1S*v^GL2i-X{&W1W84*$e}pCspEMg80sw%B0G2L@&=g_B%lQ7M^iGC(q$G3*KQIwmgr$(Voxao(~S5M?99RkcViq3RsN( zGf?l)Uq!~luOJl;FBf?WnFX&wKJ0_Da1ySx_Z^{i8&kKgx^PLp;Ge@)mQf_(?oRxlND+DUb#g zwAH}g&2s1{@BMM_g5Mut&xg!`0@wppMa<90Qm9$3lX7HT4R;vG!(ax-6rEHdtzd^1 z&U?)ZxIdcBSX6FkyU#vgH1|j2xR1f@h7)iKj18<|k)y=kh+> z*voo4gZ&Bfkc+W(5&z3jfGy=8@)HMZ?-ktr;G({+3I5W|p1^sD?|Ja+U=41+V~`JD zy`amz%Do|b6c^9BNVgMQ@}yX>-O6Bt+$4PNgePSHc^AyYt79FvK|NvY<8Xg8UT^!; zQ~Fx+wN4@aOXI|~>?yfNco#RoO;?}NJ3kB4H_w`=AE3`3;NN$I`_B1K>HC?p+A)o; zjT7{)uP5l;GD=_KcR}~=7yqNX`l3$W8y2O1M0i64XI~fzuDS^liW~!DVLXJxWS9!3 zhGZI%`?(3fknGQQNZ!*QO76QZDHwJZ8~KYAZJDPly!xDO&y%}#?|eQ+x9_S&xA&`G z>Wcq%Q}^3fi**e>W2I5IS&r>X;tn=RnwBStvn)lXllIkZI+=-_1M^@Z7_lRDpQP+F z8uwB-YSGK@xeqrl3Y3rdcFxi>UMhRXOV#nGq?-4BHGSbyyF6U1x=~U$I7V!?F;cI4 zLhSd)O7-jjse#&<0I@C!kUFG|Gkg8M0I}zBpKrv@pESq{@>vP1VGXQ<4baLxep^PM zv}XoN2eK1)7xxM--VL~+p(cR$a{+Q}XP`Ksi92^EG_U6y#aP~1(B6*Y0TNGIi=jCz zKsF(hz=?l*Vt{l|t`yv9;Hn{A=!7mXllBh$cR?29K;>H60RIAXtB$(O;2i{dG1zbO zZUbyk2i4?JGeak}*pW3-Cv`|0?s}vhyDlY8IUIz;V2122`T&g3TETs~jqzo{wwPNx zGZ?_+T zTgP1P^ReMBaAmT;&fq@sHvWWj!wEPAmH1iU47znEbxY-)7WxISFQ9JB4YpY73K#La z3|F8Zu0vlPbDMI%9v}nAyKoN%UI~x~$VcF6dP)qD*a?h;{LK@jfH^%BeGH6+@$l_E zh#IfqxTnH&m3~u%qF5F%8T_?0e z21qkFp=lD|;m*ZYppkLgFr4>gjAhrV0O>+@Vw)Yw*u>5NX-f@|R;2g*Y-2vKu=bGt zK{yO%r~*6T>aigk@#?^8!X6GV=1T&k8abITrCGMv%9%-L3Q7#bYB3y9WL$2j{Br}gS1P?1Kj(< zr6HE@1J+NJV@SutaA_i(lkm;RmR|MU0pBK&&OLYlkD#(ETr6PVIB7k=+(}ybg{ zF(**(`bm^gwS|d8^m66`$X?6%IxtZJiQ_DqD9x+*1^_)2#z6ZF>~so!jvaR)UEtPu zpBF>@DO)XNv?A+p+bXb8_7Qdy>oK-CmiXf#945n5m<}_cm9f;8>>Fbp%n6;iyKt*9 z=4O01SeSoo%s)s+7xNFglk}U*7<=oPKYGGs4(ZH;g%AyuVeEP6|HbImM7`9;>18SU z3b3zbAB3L`>QeQx62H~32G+p_h=)y(1b=V;DY$pQE+}B{p8uMkV=eoSRR(dRp8z-W zgbTW$lewmPm0oJ7Q!VvTb**z!w=U|4wA(m~R|m=|;+=uBpuQcyfHWf2`-)cVxeeNf z)4wn@Kcl;$p@(<8y?j4~bR^RL%*Re>MsJDZn}k;QEL{5ML(q7+NZOa-3S5VqFaUSq z9=IZg%LC5kQ*jT zk$vn7S0MZU=Ep}dv|E^O|FROl>O0IuP#etLI3ifqE|oXwmIEnwzbh9uHS zfi&0wl|_MK*%v6g(5-yGSc^LgJqPThTaVjjV*h|Fz|Yag{sLJH&DR2?6j=_=JCv)7 za?zFtaUTZnx{YmimeGdH|J7mCe~NF9Vnxw zvh_COFOTuJkMW1>tib+}F5GTp1L-veQ-9jkagFguzdMo5(9(nbTl}SkG&{fzr=aqv zzgV39at7Uc+Fxq#`%69_eiuw&O`e2pEWk!)V=L$V|Ahk@#+#KnhWefxDX8F&8#*4tplZz|&#rs8KFG{|)1OvvRtJr7xly;-gq zWFfj0JFLy%y!~kRU>$6L zc-RC3Z;cn%H=&Y*o&ss$pwD+8od@V&^j*py+s;C^GZs2x7$32WkM)cXq}z@NUW#50cHZCD9|)n3LZl9DMs zX@)B3j|dhkvhUR(u_N=jD=O%uFVI_{nsjPNr}lJ+Sh+l|<9)pCIC+45Fhp{BZ_zwGG;#13IA#T;LuIl?J{$Xj~E^$6`XnfoxhIB2Mbp3@uQ-iu~7- zKjEwu7n173ye=D%BGWx%f{s-4i&L7aQp8eM~_FsMMzwR^t-eCUi#XftOf4f-w zzrwair~e7Qhe3XOt~`jIZR2iVSl4g)%VFGRr~)et@G+Ym>3|l~%m@{*~Va`Us02jf*nv;XP3|F8Zn(=e8_FhNtILaEv`g#+60Nk7z zTsGD&=ydX3J>zd+7VCN}^8n+|^_oHM5-$H6#@iNtw}t)y9)aN}oEM@0ML!vd?Ay!v z-4Y<7=wo0kjE746ETkQdZe=Z0=atFmQ^8JI>SwSYW*=B@x)M>-oI zH_)8i==!rC|?-m(x_8-VcU|3B3k7NJP4jsL; zYd7sWi2XmLoh#`7Mc6+a0|zvL6Plp~`sN2nAnEk;oxwgaIRMvVd=9@LLIEu*cQNzJ@Hh zKwbJjdt27vUJUCXzm@UE{rm>>c-RCK} zK+Qjn*}I44pXNJRlFNGu2VtF)$h$jBI?iYL=F!j8y<8Uaau)l9EcOXmQhhB;Y6i2U z_FDe*xnkLWwg5#=>~;-s2&?=eW_Fe@A2g#G_75%K;(p1Tn(2K9q1OM(JiSaM{pnyS!7huiOUUQFZb1-h_CB9o z*5PM^nS{&zl<$r_{+w?NG6%O4*?ij|^UxPUG_)skK1g8SkjeanbbJ|gS!p%)vyL?ZIr{p{Ln?d$a+xo<2s&gfOyyhNzm^xND5NjqoyGTI5+J; z?t(1HfdbeAecQ4mUfG`?HiElc`ElMji0pq=#|vHNGsvET4S%s)4&!fzfvdZv3hB_X zZ$ajN$yxL{Ke3}XGrv3inQze5ooIU>cb!THH|v(`Jog`@*@^#T{7RKSoPeV`KRGg* zwiqY!`;&fB&3aYC8dZCP_3rxC>3@em{fzeU1D5 zFz)w}tuuH(Fqd_BHtTRO>+U$#U2qL&9R~GV|5N092F}6-sLWvfTM{4_(XEY~zmEn= z@#$o_jN5*P^LGj7?*p8_@h|Jmkp0KgrM!EaygwsNO4e_dcVjn8>7{geFD6kcuDvCP z!gk4F%J#dqx8y)rmV6MqO%5J?TRy!1wq(a`k(|6lxkB3gkUu&@3Lux4ntA&+^V{8Y z*>iHcT<6(M_-!x!G&@uFMyATXid1=TT&9$E{YD0OHt_Cnd7ri^U&Z%z>vu}o@SXBu zs=plU@|U|jzXuPXkMAK4?c~0h|I2YS4|~PWToWw6=lgGqZWzDY<@YT$!=1-9Vrv^Mb&aFN&U>Buqobtp;RI>8#$D}=adJ%eq&RiYNK@uG$>klS3%tLd zC}aFEPX;cbelPPI?Tvi9h#m^{xXL!^I#!F!(vzpD_|wGu4n#9Wd1pheM9Gc>>Ir21Sk8C;rzxFj`cDB zKvM?uPbTvZvLzY6MnB2r9j?nSSXPtX8fZT6Cr*o>tV8eU_G1n4lMU$c;BF)BM)H_T zUa$#2^Prz3Ayc5T%ujY8cY&qPkNL}w`HQ)$I#}{OK~g`+y~ab{bENW~1JwDy?|FXP z`U3AFUgdjUY}3?BfA#P_m^5;r0QNvJltMWS9AZBBl;7Z@9|p%*T0y;I za^2$GjowkhT5y0le-(3o5o-X_wU+t+cCgeu43^qK&gsdV3o|$uW^(>T+Ht25*8wdm z4RFH=I0a{*b&&J#L(Ztkjzyfmp^FcOT&bMDp+V#P8^-w;>FDPC+r!zmm-8>O1-Cgf zSk98p1-J;8p_240a0T6Zi8JhZ?g6Ux(vRCdj0$5HB!?nG~{p#HSKIV?m% zNoNcgNP8@@GLrhkcywzVWuv~~=##;oNZIhSWl+|gAu<)e=`a)Kz&uz8(Xbf2=M|0i zVNbji_X=1ExqP3X=7;_#LS+r^zN^950eA8ZjMXQ&XT@&=#KR^?f)p6|I8@S*J75=N zK@JpvgEe*!Qr*2bXYl(#^in7XbqC)8os9o3#=q+Z>u(?5Ci3hsn4t=+Ux{$YSN=^$S>xJL0l;#0ocL+?As`&8svxBwU7GSpHwD|qkUUuWJr&c5I@ z`vmqtwjTa(3t`Ml7`w>pP|19C6FC5uSiRgu-h&752n#a{le+o*UWO%lQ}S z!tF*jtmXU*$4JWoO$#{xPT~9uE#zyi;P;ZGw-i>uN~r8&{fE`)R-V_w8uWEw@1txF z*@HkGY{1V!`QnkAp!qiU`N$M-Qr`A`tZyZ}v&FpwTp8>;GTDciSl^J;2Vgyk z4TG%>+i+qd?5EWoV@oxLzNEtNl|H5eNVE&&Md5nee5Dt@JDolr&(8@l$Z8+^r_zvQA&Zb?#d;T9}{%1`X zI{(u*O-r!N7|#F57WyLpNEqLD@D7%=2KZ)j9&#Z>EB7b-mUhbo>Erhai*ffia36(K z=Yti>{UT#(9Oo4D)vyNgZav4^N*?S7*5U3rN!cFiWCMCUxKC5I(shJBV*KAF{yoSag`KDK ze;m*sL2IVEf8vY>9nc9~)ZKN5`a?qn<8KM~PmDPSvS~eI1KEtb1!-80UBO5Qg)uO| zH<$yrCdyd!zOTZh|0?U(tclW>tCR8gg~Mc+3Jz>&KX+o&(PzROPSK)6kiD}L;4*ps8@f9xj(FLNG8PlFw>3$h>ws%_YR7xwQ|>C^ri?H@+_BbEKD z_a6m>+XKZ=3gvJR4nr$p+h$;6Q?UPTpG|f3(r?(88#`;rr2S)Q|Mj##vMGl4kEH$Q z(*DHjdy@ZqK|1~O`F|HkE7-vSEzkjOaJ}v)Cy=HL)(IH68ty&Qox<-7c=sP~bGMJK z?f@?$`;YLO666)7!$1??uOe?M9qxkDpKtGw7oZva9`XS=EdKHc*)@(ejkU`i$C`#T z{EYDe9m84kCb51!^p}yiLtzYzh067;Uoalsx{CE{CwKYilfk=wEoc2&#QKG-=6TIp z!jQfd>c9r|V27#1nGQ2y4*2fh`TbHO`y1S?F`QdtS?AZY&LcZb*{3VNYa-abAKy#lz_4k)z^bYK}9ZJ#5!JUEq>agG8*zY909K_Fjl=o9gLM7v- z3TXw4jdOh}b_I6au4mb|FF~J;9VcMNG1zYp_N&gjecVwIMt%R&fmGlBxRGiuI)U_l zf3TVTKXc6~+-Kk{IQH?|3FJk{B* zDMu~kQs2tdEvH^-Qby-n>ug5@Uh2H-B_ZV8fm$OrHUTCwLg?7AJ)w|<@2dzVK0 zkEZ<4aGLv1IL2J(fF^V&G!w2R1N(oAv;PO2!}$L~ef)m`!wT#RMnWizfw3?i!r>P6 zAK*6`17BhrxYgRL=0I~E-|QnTMS7Wz|4gWKaxQs+?+!i*5-asnb+kbp*g^G0^*+wO zC6CX)HJm9qpV~N|)+6nly(g2-9GC|SAsQCLQdj}48N7eb0{u?`FADhtcEqP4w`2W1~#B~q;j{A93t`Po4{R0 z+G~0Ly^8nWoP(3_GviJ{ra>j=;~mIdU_q~)#<_lOh-Bf;fqL?@pAVI~Db!Qr{dX7j zKTkNYPGkOG!2A#O=yv$~^K$|5_dqeY(yL86Kenar*&Ehke{k%! zvNi1QA@+y-_WPT1(opa0)HlBe(H+Y}=$IKZVLE+|oD5T8I?RMQFb~xFyAT--i=m(2B`rnv&1Vmcbi^8DC2}>afpyRvhW*0^^p4@! z{{sGJ5&9+=`u^`ed!^CXzX|)tW@;<2|3+-12OH_dHjwsi=BIA%2Zl30aUY&U{1n&$ zE`Ia23)xEFwn2L$Z48~XX&1QA-OL9K>@6DiVgGs9e=_~gv!U<*2$TOb?K6xyn6z@B z0QNvJC_7j6?|Y4J1aSA$-sQ-Ha2U*rSl*!`tzZWSc)#BaMK~M$^s;fI49r~oPo1&0WQL2XyqK+cAve#KF+^+tck?y zLb|~1;x{Y{Kn4&-oWxz;TE3FZ*VblX(oTkY@7@!t{{_zm4mdP1@7= zMiKw(;MVZR-8G54kT>x&lTSYPGzCy;!7z1NrJhW0@@B4r9`hWVL{vCS%uj&fk_y1kg zpR`*jt7ABOBhr}+Q(-zZ>!?5eGtoOV>JM|!=Yg9(a-~v#f9ekl@iWKp4H?ovSu2-t zRzzP47W`|etJ;IDz`YW@_d1c(a}o6f6K7=g8=u@le#6L_Ilm!73iv&ci{AmQA^bYn z0P(O1k{|`rUsjWFcb0d6=2eXkp2;A{2^FY=ox+-IRbLix>;zQ6H*I&fcv z%WwtqH%3drW`67c{3O2rCLVs@eqVSU{U!`RJNt|d=!7nCfgA4f>>fOTM_~9lV+}^a zz&D&fHb+Qb1mDQuR^K0vL5_v-5DsRI|KE(93e#aGR4y=xWj60W(XGs3wcy$qB@1z@ zIjtVlHx6}B9Tp`uQ=+7HMwD2Qb&*kGL)PQABTrDK>_L9x&;JTA!W`0w2Jd%ChiG5I zwat!_cIbf4^HH4bqQnL6?kH(kA0>^GqU6|uD1IvtB~8nzfSyLgrb&C{YJ2U!5Bob8#^Ye|&s!Cee4%H{4NZ|c>BJe)aS zewVO7D(=jeL#v*bgKOu?hpXnwfjhJ0gW%axo-kY9KR!#!*3OXqF*BsJ?>Twz(ljYq zJ5Aob^Q<&k#)-4{Nhu}m++m#8OEhWJY2J5vj;?XO{f7w1T4SIQ>#&5oCCr3;DCEgh>7$wyS!=y$R zB(;}#-!nW|-kJSF*>iM`93<_-@Y}XIQat)cvN!CXWM9lb@!t6#C8uUOd_Y!|3r90<|75{-J zqvZXVg;L%Y%{$wLQnvobQd1Er)kTqFZHtszTO@rFDRoC9xkHcS{br=NzVs72aUIYC z9pHu&a0=SiMoR0dNZ!3fN_#>i?_MI=7e+K~C-`=~ML{(XbMGhmkPw8gsE)3%N^DzqcIN8|7R7 z{}v^oJnx?nC1a85{%^c;Kh3^>&>-RH1+-KC7C)JcUbz8-xEGmm>l)ZL6IYf>Ima2Br_?Saq=lWc|ng zAbLk5Y1i<3baXSQZ}(idyN;3f3kq8`J@Q7oWXTUFSc|BOdk&|I6Ooy2;bpBsY8^_WoG4u`n z?ERewvf(KEe;fOM3+>a!`M-wqe+B3NGWO@31^W0+W#4Dq0q`B*|3}*W2S;_DiJpH6 zVlabcgiA*u^WQ7kzX|p)S=Y%fChLWhR9=d%CF7U`zO zu+62j9TnJ#W^w7s)*sL_JET2~{X^f6+?&$g<{V_lo}zG2Tx>a;Ig+TNA0^|cTqj>k zN@*hU385eSJtDwLT}J}ken!Q zi{){?ye_u~w79#tj|YhSZhhxVhb`Zq%so3K3?zH_JBrA`7>eO2#%OeZ$M-MT4_z;q zk03lA6EF#n{U?KrZ3$09&$~sTgq(%yVIQskooReu-@Hy+mC%;fZ_vL#WdEaghlK2F z?2;$+FGg60?mzRe0E^IiC)(r4_y?KU#irlp{v(SV8m?OZd(-+~GR5X^qNmSV|9hUl z@qK$BN@E#TAa&e&0W$BOyQTcl*J3^XVEv}&kR9V0cotPP)<}H$6JeucqWOxO$=HEU zgi>-l>bH20UUsiny62u*?FR3exC+1R#BS`xejG&4FYTd7#&8tfpY;BcUG^hPl3Snu z$lo$V9vjSglwOPYil?p7(*9@YQQp$dm!oGsx;H22Cy_hif8O2B%kCAbwKX+q^=_wn zm#j;uhqcf3+UF#h@}JWgj9Jxq%+sM%6y+@6^|K>G4iJ^mPAlpZ)+z?RUL?L*kG&9DV;6 z?E(g(2!j#r^`+YvwqmI8a1^8L9qrD$>?L}BeZnu~|2_FXFaL;NmH%G(kID$5vNG1O z<1qn~Fa^_4f*EL4p4*i1_N(guUCQRZkG2284a!;cU8ZA5q3MqD->dwSE&i8UY79X- zb1)AJ5c%fQ8_ap2M{7#jv4p-1xvAd0iO#XZxlsAM@f-205S9DYJP59DWnF(?5RyDuj1pH?nNw{p3L$MhxY@uxD0`C}d%63*sF3$O^$nCTL78CGBw zqO}v1;#bojt>0ct&sL8J|91U$^+fB>QHwb0mKv9yX^elaacZ)!|Gn|PJ^k+#hpoNE zVZHE1~=D9*_b%lDV!y9i><$(ZvA~f>+cJ#zbBhU8e2Dipqbv% zE3WOCuvvPg*p3P``)}IV_)dDpczOH1XTom!UgZ3iz33cgor~NrE+(CWXfe?5Kf~0kbF6&j!f(`{yt9rQM=GFYs7C5zeWDZr1ATd|Fj^398TaQPU8$( zugm`(*VrrnWXDa{=DbhzN@p!0ln|n)W0Z`M%Mlx zUE0Ed!rj3fg5MQ~B6`=0&xD?zn?vx;;xJfvD2AgLHU0EKm7_RWi$tM*DVdzCpNg_q z`-Rbt8;kLnfJvBwX=uCd9qdxJ?0IK^e@OtvI_I)i?9T>(v4qL|MaSVq+5^VdG`8jZOuacLB!Mv*&h|~ z$LJSeng5mBcdr8Z^@Zy0xSS_-V^(1`dVcQPP~>`SjKbgbJp%P)*9Xdo zaQBF3LMa*9|J$SR7t~+!w&R=sBUA|Q#BMaJ*VEWb&!ivrP509eBC<)d+q55Kv{rn9 zxWkCyDB?&Wg%(uzF|Ql78_e@&d)H};6WZkZ8vO$_XzLr7l;}hEIh8^sBL(;$ndge|4mF^|_736yT7y11UcCp#S#fIs_kT+0u(f9*-8&pn=M|~T67j>uP<30ALK4cAQciB(;kp1AuM5lGSY5SmFR`&d-??Pn^15t#*7>eO& zHU8U%cD8s19rR9QFDnDy&xRWPUmTNWN}5gXbsEi!^?#9l&iI>jMq@0-V*;jN8uIhj_L~we|2{kNSLW@|XCaz%JBOTy1z3dq{u|<#&@;~~msmz$f!s*t644yo4sCP% zeQi5d`E50#x$*T#99Py(@lREnyCZ%*HexfX4)Je^FQsos1$JUL_TuCHm;J&AaTsOV z-12{M{a@gx6pkay&z~eyXhE0yE<<+v7D0|IGk^XB+4C>Hhe)2r8JxofL}PlF$Sdf^ zH8d;ZQUCh}Ju_YTo7^Y#&~GExtNbCFBhZ1n;<|okUnR2Jw*?-Meg9X_8v{{Au*XpBYA)5dR>DnE!_<)c!jqJ5vn`)vZM4zM$-pA6-E2yyj5ZKt~DkU9v-4eB6N zv#o1T%jS;8$?KkH|IcUtBZ-*n87!?yn1X32!3@ko>rM0j?|d}>f0r@&ea7YwnYT>l zgc~NZ|EC)N?_>NQP50RLXyzAaIdA-Zlk=l?SCMv_U4DpN&c3gwC)fWhwgpPSs4mYqz&MzS8}x zo$J0XwEo`MWHf)MUO4HV)R76tRyb}ac4II0;~*xv?}y21=cqyLFzKL=brAo5Q< z_Ahv+Qg_%Fo`2eNeKf!1Px_~BDffux2gaP^DB?&Wg=l=gg^cX^XivXren3VzhZ8u7 z(>R0Ft-^4Qynsu%g63_7A>CCNy6Ktu`VF%`5w6j1Am48gj>f<`$k@cf&?7$bAKWGH zqw3|tP&ukFJfO$-=oh@HpRq`pKvJ7sFPzXfiPivaRkua^@VqhCnkK(@ed|-kiuE;~ z7#O1UcSYiQ>|5CLnNNqo^zyIT7w*fxT|ys@{P@4L{=okyT)#uTFtSf5cFbsu#du7> zBuqhBYroLyUbk&g#>vb8Wm;XH-=B@1>*M*T2O8B8$H-Thc>*|oOH$5!;VB%-O9vjYnzi>p@y?jKt?K!2`&#R+z zYa3UPYtgJ-UQcdBW}kKxoeTL0kV6)m{Z@)U8eiC^olF~3uC`CN-zyLs(=Y5KE6J+o z%}=B6#eN*bVZ?A0aU_vK3o^(dR?{#1^yec_O>v+?P0+HYs@5AO{MyU#xvzW4qo!>*Bo!Vi`ghP@jK?L!&D1;79B zy#dxD7KFEUJ!yPvK=|>?*8idWpB@BrJM!Pg-MYyhtfLCT8|sTA?`c2$FV^RG;U^c_ z^Fw;WyXoF=)TW(PeSWL#|29^&kC0hU_rr*hyxEb2sfxexM*U;@E7yDm0nv zpGGsi1+kF@A^Xq23!|kq7UNO1xFA$cE(jCo@ujW}lju{BJYOUv}_aQ*X%7GYfk|J96|B$Q_XO9_N_Z z>lyZjlj33{d&6n+45}vghI8ZvR8E(#LGp!5!dFm#{=<-TjdgwG6V>lO2sQFm8|7`3 zcqCAdB)a|Q8g9Tn+><*N4jJhy2g-dhbK? zQ{>+@G`oft$E6lO2z?jpPb!c7$$@A-;u!xeqG!fP8-wXXksBoK=bZx`6FKZ>wUQX z?}@%)g773v!8DY8-uVCL3qlFK=gS|48RRTPd-u#CvtLqb{cqevaKMvwBdY;pFB4gLt66B-xe{p)=J~^O_(xWwtwd#Wuy#;yyn|$T- zxyN;j%i#o0;xx|S9D4L4%ibw4M))9H5biG1R*+XB9oLZ3X51im{PVETL*7O-ww%7l zwxDM!^-UaepZ);3L*4^)itkA1{~L3yDP*S(dN}S{H!<>mv#-Z#f5+I%&VSxGW&Pjr z{$Z=}^1lC@_Y`^m-|yV}o9yd5+IH=JhU^gTod2+GkH%#ima@-BvCpTn&!yWmjBSnP z%k1;>)(;%8?}_w^Fc?Eo)oA@doAm?q$R{7||5HpKjimoYdk-YUM|%;B6_;{d>ShO-}U9^<$*=FBm(t6>s9yaR|G+r? zzcHTqbkBa2cL34;quU)*fwBVSB|I6rwI4mdurJ*wwIkwtzR$;@|Gb;t^{;$N`!Xb+A5(ErDN&H4WcoWyCI!8u&O z|6l)Ke*MpR*M`UIfArn+<3G1w4S&?X*5A%u)&JMeZ(OSX zPo|{TG+Ei4s{c=K@jSMA9@{ECzl8q2c>oWj*>{ODh8}Hu_Bs0xTK_jtxZAgbi^$0T zH#iDE>77!44y9Lo$^HUgF;{`!e9Ad4vt8&JWxxHbF$els9rzREAKiS0Vq+y&uP$WG=L7{gC6o1b825&M(l5b`UAI%=^G?K_TCWn0ctqm{NUR9Ktdm!#KS{>7d_UKr2G`hdY zBHHVv;THejJ^sJr>IXE5OQRVrh}NW>kxus8%D*=J9KCsvKI$EN{nInkwX2iun?=8Z z+y=*?bBJ~p-Qr@0*!ScOR4p_XNZv+eqjI%Qzj_nfUid!hCu$c_H(y^C)l-e{;otRt z`cL0)C=VEjA`He*49CCg|McGH@E`d&?V(ovNROXwCLC;i%3nA!lF~<{1?-42-vlCV7g5Bg^ zL~BCc=r!g}ZY8(f(N>DzkApaj7>=U6ReAcp=Tpf4KaBr>4gbG(vrfC2AnTE2zc)P3 z|NkEUzy84=t*>*AR=>6BKeS&oKLC&R?avhpF zj(!5Uaq2{Ion*&kdozoRY4cB$XHZ2yM_xeX9@o9=;dNgUzJmO9%S+ul`5>!@D1Y1B zH^fng#GC5>M)wun{&NjC(1Y8!i~D$hR{icae%E#~!ynT@b_!?7oHE{UME##o|5vL2 z+tmM_)$jM;_C3U<`d8|`XwK(W^cNn8B1C?~vbs-%!SsqJl>P773!HvJnHWwMBQ;yy zPL9QROhB`^G$zqAJ zSwME1|Fb9xD;G>eqRXP`Ac=wLB`H`WeXhEr%26Dzo1L+5LHQBciZ- zekKaDF|xlJ8P3r!pob5+?04oCm}77$3V*qOxDp+QYq)`JAH7FC^k_|V`4`Pc_}PeX zSNJ|2pzr^#KE*&(8zZjiXZ^i(^l`GTkM;KhtiPXX{XK3wzQ}QdF%-j5jL{g2)>GEs zpSS-0to8Ty-tV|)|NWcx-#=jgeKd@*{{DIE@1>Qp-oA;RcCKdUYB8T6wWBzUm(B!C z!W1;W=@?9-M`^Zq6^9b~4CGEpTU@8O4(FLAE_S>)%pvEY>iy!dfLw%1dTphAC5pom z;bo|&CwIwbjePp8`nvq1_MY*7)X@{DpYFMN)*qk$TlURQgcZ_Qg$n+a^0)1OcUJ%J zqB7Q{e~mwy|2tRtU#R>qRsPBR`tMu%f5^}OH3q+0dTX&B8`1oVIReX#`O-6cr2VG8 zJbgRz^M8dqw>cMC5yg8(v8m=u&@0E8D?#o>)hP2L$b*Qx#@gfF!BgHvGPy+^M;`WD z3`e75kVFbC$RLLkIEm9ZgA2HXsL#;#OJmf={JVcu9HMpik#BPwqH&Qcj(N2HubX}i zH_#LDIqw#E7x(cXijz*?W!_B;L=mF-|AWb)7>;7(=g^Sd#vxPciLvzYn1D%W78mUS zG=-iYe;i|IdZK|JIm$(CJ#!<{q$n(rETuH2-&=bQWL{mS7oHU=@1)<(d5Y zhpumUM})i0Jzq<%$3|>MH2=4h+>Q$D#H0DYyXo=g<&AG;FMU6fL)cwNjADn4<9lKI z*06nR7qWj>vVXU*Ey?^kx#ijf|2c@mh#}v9)o+aUzdb6Pw}08d?e{*i|Jl8rhdyTi zcCmkP45?oBFPk^b_Ko(*Z@DP$vhzwOi44P@ZFzb{t-Qi zOa5~O-MEGu=)rB=Me8p1zdT1{8<|Si)alyRdH%?+zjt36-79el=~oc=cnQ_%e{_WmHdtS=}bXQ1af$Nh3l zm_?8F;!eze=<7?a^gi}c9tU`57dziVeTokX!yL!W!vbV~TNoCRORx<2?+EBGWS8^# zq4g$zJiFr2eq+(MP}Aqxh4R-zHn5lT{&?Y(aJM#fg><6taI7NBzGhE-axK>5$n{Tz zcZTrOm~ZeC`SlIguu=SGlwvz7u;Ycl2+>{yJL%~I&PU%(FSlP>g%2*a?}&Vq`VaEd zxs+d%%n9#z%t0JR3`Y^i?#nkqwEs``7e%4V_v(}4y0>~>`Z?S5UEV-S+<`U4;YaTk zhl8hvhaYeHY&f*xv*GRchKIxV?04xs+Vje%!}l9+guQEqh9A5?H0(P&IK1`Br|ePr zsqn-12Z!wof9bpJzYKe&)q;v$BSZO~&x9RQKVuKfVPWT>5x&tkB4qrQ!|okJ!ap1^ z_BHjB;d}c=hQ0Hj4nLUrbU5L+li0u99+(@R2>T8V2yfjRV(iO!Vxlmd_S+ep!v$Qz z+qa$!N7fXEEA&{Og7A|a-q`~M;oUt2;pnix@(m($e+K+js2=xM%BFQ#eXP6c^Mz11 z^2LxG_XT4#Ur@KaVEpR^<>=XPtoH@}U2E#jK5MR-@`@Yi!EM|{)W?qI9KCUoPgJ;i z5dQ|hMy)vm@rC>wdwWXmRVBF-0u;nFF_XpBYkEywtMJU!Eo z4}*_o0(}y4d@tEPd>2FbFfc`2Y)?^`MwXx|T@+@JvrxHBz7EOP%jQuC&qMu8`I5)F z`SOYCZT5deE#jy{0`-l)RdK|gnEtZ}ORx+p@J5?^sBgU$t&{D)Ih~JReUaJc{odvM zPJ6$#cR8|Qjq>I_KQ>eQryXi?Y80UF~qCtllQcF zm(@3}E&qKV*IVtHY9_01T-WcvKjL?7bmXJlD~G4i;@6z@HqWAqjwA1&Dzuqzay3hD4wyI~g;SF|n zYL~st9TUx+i@qT=kY0qrXrIfTpUGxl%65O54UZg#`fWJ+O23#KjjHAHg{qVb8? zaXt{|nSyC3K^6Z)TtW*&nglv4Fk^<-vOT zpP9c;UxpP}h1FP#YVCE+8t0IoKjnW|@0g9)j8bez1$JULTGWKFGll_4v{gO1Ay`L@K&y+Fxz0%o_gE)+4?Qa?}`lI*fC_Rp-OlJ|bVI9bR z({VNS>h)U+Rmyq`nL#DJcA|3Soy!TA{X)M%d5HGauN$TuBKzGJ!%4qA`krUBCN-hI zUjMxQ1(|Q}dezs*l{tYGs1#dYAu(?D>YWK;DRS z+{Rtp#{+bK&$!kK$G=z*`jZ1uguy8L9RE5w9K{%oo>%*avE+E9p4UhDl5fbzeJl5EDL%dU_Id zNFYD{HQ)Pz)Si+McTC#)^5&T(AxSU)qISDLe^5BH)b-7EebZgvIM+9-Bvj9p|Ao$j z_)6(+k?x!F-{w3>NuvcB*h{yiwejyYIXIs@z7)l?GKN$Z~=ISOZf68DznG`N|%xH|ocuc?~Ou;m?Uf177 z`bkM_d-~&Ornhv7Q-_pDr~Fy%^gr8=pPt{ptFv#I zMbAj9eUCZD^m&NB9hOBWI=0#OKRg{`>X8M0TZAfUFCmwql3shBU-7d2e}q>dnlqC` z-Fxy$c7N@q(Dm~Yd+fU3FWNi8F;V$iORmR8MCEfc8I6DTeDkGHO0PiAmr5S)Kgi$v z#umQ0V4l~n<>PJhpX?pH(=ofT7yEG#hY`b3#L;R`f;O}tu!q4Odl$6X!vNW{_Ao$0 zKkrYW_lHc4^!|`(;byYMan*;szX|V1rT4efJ3%Id>&fI6`v=tN6CmkaDYPJi98REG zdtQUuaoQh5W6IIqvib3U;q8t&iPJcPbGU#@xPrX>yNCUY$M&yw{!#n>c>mvj&k>7vtm5=o=Hp#zm5`9-HOBjeE z48~9lM=}2A-``aJhq3>0Y@7U}328K=1<|(yM@wfc#$y7S?>Xj{J%8w#%kqyY^l8Yk z^|R<)$o@x(xab>7Gsszp_D-Hd&O`JKrD*O%91DaOp}voNA?mv%kpKRTb|}C9?Vl+B zOZ;aUqP-7Rk*l#5>#-4kQ2yDt9fy=n{s}G$G2YzJIURs>NO65z4Z7+b--C;1=0HV4rPCf zx((49+o|KmOUKOL`s z(4QQLA`C_}=X)qQ9K{%oXm8a#Q z$L9azL$!Q!owD-VfUljV+W2w=OomkNi0MCN}9SY*S>18D~dBrTPCg z=Ko78)oA|zI<_sM{rOu^X>2|FU3I;*s@Tgj$T^55jQNw%w+Nzd5+oCBa;$JY3&h98 z)kgMnoBivB%l}$C|D?K~9_=ldUR)HG(K8!d8&=R)A;(>tR)rP=@dyA8MGAIFe-PyUZvTYzS? zAa?Jmuv0p_u^0PMHOTyfVNZpF^k_{+Ee_LTNcMAWNH`|i!!ZB84e>li^2bKUN5`P- zEqxG_f6;egerc>rTn0Iuz)74&^_%S9J<9(+{q#2d^7H!T?BseR@0c6lm~*&*ONiFu zUm?434Xw&in{w2S{P#D=&Qr?gE#(spyV$=owf}Rq^<>jh?LXNp+(Jh4S8hnB2e)w- z%`^3%^$YIPGuxy++c$3MeOI}D;p_qD*z>Tj(O+C_u{DQe5vuwaFC~YfQr%En7}@=O z!*JnZMBn^N3fGYdeS@w1K-+E_7gt_-TJ2xpm@f8O*^i8A{HuM9g|i z|IY71c7Mydj$od^Uz>+4oa)qeHP}R{bl__bSRshb3Gd}caQ(yZws&p zORx+punIlDa2>A}gsy+}9twAlu+JsA9viV4v59;jWZCD<112j_HO~6qQJ)Aq>5tZV z@22lZe%;S0<3Pua1CiC&wf~5|@ft@RJ%MPBS+rO0X#aWR{fBb}qB#S5g%6_qGY{tn zw4UYvzNn4Tj%CoXN85Br8?}#b{!R1WW^4cO2jAblrTs$-8oJo;I5vR&j;3MkcQn&m zkl$aRko_($gAQ?>c(lLoVb_xKZp6r=h$9)rp#{~q%wJ&7*Ru2DmyP?M`q=u#BhER_ zbvrJH6F7Kea)g1k+1)PxR`##CGrZYhG{RzYp7iL(035m z-rf-ILH#}V6?I6U+J9@<=(V%WKk!b}vE>tFy>}!@-u9o5&kwjOd>;>x{aT;Uy3P2v zaqspW#=j4+|KC)%cdFyl#!B=_8>X}WyUhPV3i&Cy?mo3QjQ@$>mAOBpm{!y7N zRKCPLn%@(BGpGMwYjaTM_ad_Ss`9CB7);OaoB6W!H1y%9rzdACU-OkO6pM?#%{iJJ zi>f2W703yQ<_Fc(*n{DSwU4g1)^#T?TLX#YEq%3K`FL(bnB>?gn1&L}z%0x`+ZH}Z zM0;Ie{H;HGxc_R@9*>j9Ui*HFe>A`368#FwULEQEQ2!io zZ{Bo|=DRn@AvW=uus%u)*Kh+>i|zkE-p1fZCQE0!bjXfD{x{Mb0BPi;RV2;9=Au*XpBYmS>G2xejY&Q zzxlqvq2dsm`%D<`xCxkqDX0=xiD~q>yw|?`OemqxKvI0Ya6){Yd-U=4cb4DhU>?ev z++*Z^)h8?xF8{K2_ysm%hrPgsmm%Av{O>aVXrJ7v*xblw-I`9Yk|9jqzI5v*24^7kc4afX`{%K|n{}i@Ms{)Vr=fBK9g=pRU zPVu|37iHeX=1t1~I%8MDwc46!kEw(7!^nR(bccS#0sRQX#Kjzcl#HWFdPy>c%E|T+ zTQfFn-}RqDd9`-b`&EA6FGI!q6GGXbzYII*e?RI|VduC{g%-zW@DF=F9d?iWbokz& zPla8BJ{^9r?z3U<)z5~U-+JEq+wj9%<~&?|CcHKKvti$`XTp!8Yg31v^!sU?LD%O# z72dw3egE92!{JfI;UA9|heHR3hqs>}?z?ct(K?5Q15<~FA2kjQ``;WKepoX&?893d z{xa;HI>Gw431Ls;gz){zcM>lls@+^L<6(DVT;5%)k-#=}#82!?%R+uDvBIhdddo z?mQWe+LN++l>H{(dx8)BJ0U*cJA78(35k&#Lj9(1^TT{QRA1HqxTXJr`1AT7ll4EQ z>wl0*;aSq3gLznhMOcDmSb^4F{f}MR=pEYegf_WN+l=TN#korTi-o@bw$%6E$j9^V z&w7WE-+y>hL0Bc7)mV%5Xzna9MpF*+l@zVrk5f&MBihoKFCCUk!=NGySUi& zf>1&3MAf~5u$$bAN_y@20{xDHuwVEf>bICTIbT2IJ?rc?n1A?+{t0R~X%8;y7u?Y= zkf;3qgICq>r#wGI_SRwlFaNgjWcA2VdK|4=$ z3#Y}!(gooRc@9-m`-KbSB~;RDM|!Sz`h_dP-Kan7xnB01jXNZwXWwQFPCp=iN?)B! zs1xe&X#bCE{&NF8xQ)BGj|b?x`lI>(Z2ZXgnBV{7&_f?Y-cAtLxX>6RBHzL>M83x+ z?|;j6HVB!TZf|nw3`7xzBJw??+t>{B$mbBP?JNH-|G_(jAwPaLk^g>&^Pt!xpRJ zkAl$iSL%z;71+1x$uQ0TOVCx;C(Izbe{NigoP)BT7KYR*_5?kx{fO28ETAvK60~be zG70T}oAw`Bdajav%KmSh$v%CVeTpV}dM^8P9s3lm+t{~gM+P0}L>4&=mi98Nz$(N% zm(}E2tVgwdW@}KphJC$~&B~rnko8C|WdH84hW>f?U`WW%f8FRmRpKi5nSW1@cd>uX z!7rt6NAkM=ic8#)7AnN0c9{Pk+5aOvQ~FMEyOG|*_TOdxzq$TCM0^MxPI=Z^^=Mg1`SLex#vPbA~wqIm;Jvis?$?BJ#!^4gOj6@|b0 z=b?qnAgd1P`GWe3egb7L2-lgPGFAJ4ynnhz`=IT7{C)Z(_RqPleL?#v?aO=Gm&@9h z#ro)?B}F&0p)yz(Kp2|(XXHz*U-Ag_M*GiL zjK>5_!W*4@=Gcl>Hh)_}|G!fIe~teC2K|5iuq>Gq{*&u}I@!KyHgJtSIi)uZB`Dj< zhdsjgHR$EPea(2BvF$qLMR*<-U=fyJ8CD>gf3k{PjURaT){@Q2Uo`(@Jw3BU`QG6B z*YwTEO;x@pE8h#1Z^S0^Uy0w23hYGH9{&3=)*RB~@>rYZ%ct*0@}9csj+C~yM-pQl-^ETSD<}=o`ho6${j)GW ziu*!Am_YWtWp0jlA^JvIG*@MkxG9(x{q}+Kz3cVRw(oU)oYzC9@^wC~*VRw2tDjz1 zKfN9rR=yq@*SsE%ZF)VV*1aB@7QfE7{(5M3Tub`(P`&W=P_z8?P>VR~kU%|>C~=;Y z`!XYfJ{q$y2lG%Z%^K7qj=DXi-v3hm_fr1%(y+iWi?9UCumY>F8f($Iqg4ODRR6z} z|Gkv|y_El}l>e(Vf>7f@BRC0rt}ppsrYMBjcSTU+=V>d))jU)Im)r>{@m5Z8m-xQoa0 zGpB}dU-$u{J&5|R)&D>d24g6OBU*D*Ojhtum48qWM$>z~YaTN>KKlLDg3$efImz^{ zuUQjCPQf&kpqlTahQGeH(sg#aE;1n;txb;F*=X#1hT~>o4(4G27GVim3(fx+#P^8D z>#NWE-(~GP*`OY1#Ia58^*Z-|vHOqasqX(|anIYcM>;F83ajyG{?J-_)MszUM*3#_ z+x7b&|NgZ$ER;H?=Y!JF)8YN4cYRX5@UQsqud^pgLq!z#mDj^gvitL|hrZ-`#Mz3q z$j@sdliJ1TTQTM8%!+5#(>v_9C|^XlAb~| zCXn5x{Ukf~Y5&IYf1`G){sA6;e|M?>uuof}G@}3E1Ww{K&fpwcuWSF&eoy<44tgiD zOSS*n_s9L4K5X5Q`Zw~HW*bM}I&8UU{$Hc|@%fMXe zp_jeuoj9d$a$FyUtUK>K|LhscN4MXvp}9tXg6u)G4`F^?i2A>rZ@1z}?*%%`;yFP49{dH(xyKj~{=@l47rlQ+ynjfc3DF+o z&3n9mh_T0uq%#;pF&tHF^Gaht#q{_}WkdNLO&^P-@=`A@u|ZkEcySXj2~#i)C76L( zc z7-Eh)ia3%;p#|CR6o%F{%6~s)R6X$6{u9m$=g5YO#y@e)ajBK={}%V3Y(~pM`~RDZ z)%8_2e^cZG?Ei57-vIp=^ z*q-~?pvJ9}ci0bm*ta-_6dv2RjqF>rifcQ>KK7r;4(~87--$>2^Q8tn75c98K4Tz? z&^(!q+>dX8p4s5P7)l?G+(Nzwbgq0j*S=U>Y_9JVl4DV|?x`@IoPf%gUH3%Sjk0(3 z6;VIPb)#;Se4x5g{^Y5ajJJuuEdIFsA=+zT>&=IL+&A2-Nz%-(zi+ahmOc%wr#yc| zd#q&6eq`Ud$Jqm(|31%uhx_B%Ki>b%b59HBpTFZ$&p#DPq%#AvFbB;q%fCF%qi2-W zcHsr|MaT(fmElgZqtU!#aj{pdA17C!s^3##6}cLf@>we!mxr~&>rsEzvlp(rqaR8B ze*HZ=nm?=VkiIopk7QcE-G4Tt6x&gOo!E`NXw|R$pY89^@IL#$lKsz~PoasPPAK2{ z_$|oy*9Upuv0r*Uzfm^H!^raIbU&>OYR|j2`qqiKIFhLNC++=<1tHUieesI4#kC+e zz;hP&c>HpudLxR*37kZ;xby~diRou>4i^x6Q`sRa4_N<1zd|oFF4|4j?odD6R6pE{ z%8>ejp1f*avEObW`=x%N>?_vf({JN0?&AUa{*Ct=1M%_sGeyE(pRvB@8&CQEC)8s@ z#SKR>MkBvJ^SAnivGnnnfJx|i?WxdH?fu!{T@;>%63jsJMDN&;K4BI;Gt&D*pF^LA zTo>E0R~?ED3>LQli?9SS*Sm~dfmNv9rG3`E)gID5UDZDI>LZ-hJ|T&QM*f2~{)5?U zX*4ZnUz`8ejOhEikH7yjRr`$m_kYG|H-~AT5$*rK+IiMuJvO3ho&9eX`mP*3t{)Js zO)RBn-|9{EEp;;56C1nRZpNmNK{8m<3~_72{CXj9lP?SnXs=o>zj6RkO;M{6FUHH=3i z9m&P6OC0Va0oxPX6In%lg6&>;n-Ffet2YE*gtb!cx&c{ zu&;4l_`#74_Hf(~zJGO7*z^9T@b^pqChVN~Hz6tQ?A8(CAJnzGS3VWK*Y-DI7yXT? z|1E4s%I_^GU-!46;=tdAvH|}s>{#=+;m77WU(_`^!+@ z=eclXi|_FHZ|ul3;U_i5Wg9;m-n~;Cj$R!es_13^%trsph;YU+=Wqd+a0T6H?DMQP z_%E$Z`A^~4_5T#ocm5(Y_4#}#`}W9i&F}g3y+gb|`hHRVZ;0zbMXT}s7e|J>^!s># zzU!4^3`7wIV)p}&I>m76M$R2YCCy3EUl4`) zXctA{I^#e30g-P!y1t$f#*W1=L$-;am(Hj49-wIewZ+7p~`ZH_k(VD|{ ztfy~8uFbo7@u3g0#o@lk!m_&jP6M;#KV zM-p-WNg{<7WRSxNoJ4EF{D8Fc7~9W~9m1Up%@3IGJadh2N~00SkU~?X`2{uR7a03* z+2^{(**9O`G@8@sWeKOH8LgX-*3qA#$5%cRYLQ?6_d9Lp8u?!?{~P2V=Nywd@JzTs zUP7~a;0oD|bmKFjeYSc5*Mx5%J6*p(9nvw?GfdDI?#zMa%gzd{ll?QBSOkL(58_i)UPAduOs-?u^^8kjwG7foTJe?Ec&DUm-F9eGa*8JYgWJuCo8{*lUgcX#L+{`cVAQ`aeGQ z{QL^_$8f(FBU=9#`Ts`K%YVx5*Z&wxACC!`gejPY{QAFL{QrmeclYU+tDh67S7wvP zmH&6lvv%AJ%)%VZ!vZWq>n86X+TUj{zNb%jS>KM#BAV;npblxovDwN$9{Zo@Em7PM z-;a^bGOWNVG*9*ZOtg;=Ju_4LfwlDY$Q|PULnk`0<27rTv^5+3wi#7h`18r_sNA4j z^--=+A-ogy7nLj2A%W!S`~Ods|8>egTG0KoXuQr?|KIT;b@<-2G|N6?FTYlH{fp81 z=c3U6Z@rf&LNxbhFgY~RzhjR~vKXTg%|9MXKAL|#o<0GS5Iw`pQqPR+)E~%_(fs4R z&R4m`dTD!D_jTM9OhXA`*5=pkvmX4A_26W3oca0UXQ5U&zRR40tM=&=o`(fkgl2K+ zJ5Plr^vn_VyRq+O^cBcevftU+osH~wvU&*ndkj039ULd?rnA4vdf_CQw+}z+|E%)= z)mV!%V=?Q=R@c;q_Nn{}=%9BZ+iRTvj`asi+27JRCcWregH6&;k23yFZy9F%y|O55 zlur4x_R{%^{WIxR;wr79FQv!3tf4eVki2Rg!aZx}Z&^c6?i820uFp*FMe`ot zXeSRMebsfzXGT5`3&-$iU0}xn*NJM^QRCY3-(Q$2|E@i;Q2t$aG(X~~|HP3*3N6SW zhZAVM6o!-1IgK+ohpO}T-)Bc$ zpvQa7-MCX2F43cBTvvcKlDz29bAn-9}G%vhP@qxCyC$o%)W z`1Cr&cOrYySU;+F*#B#vF+Rlk2I`FWMf39O&$9K9w|`%7>^qY}k92P1F7D$2TGyF> zfOcfiabEqW4$P7{bz(!B|3R9^w($|5X};@2G)CEi9`#J$jp}|3M49o=B62W>BKusw zFq|yLXmo#3|DNo6*4h9vrc6yBCt(Vvp-P<^)qN%OXzg>f26_g47Lw=PKO`Jehi>m( zSKtG9g8h%Jy&ttF`u%epKM#wr1k2F#tEa*Wa#f^%%QIbLoi9DV{{P8-VJ*E`T;yX~ zPtV*mHjIt*&B(bgS@mrvKR^eWZ-3aA`R}fFYS-DR`S0tH$zlD&0{UWgi}T zW|uJljHZu8?qy|DdlAjS?jWnj=>N!TE%|u-V?G;yw)?MdwADNOsDClu|0kgAIcb;! zIECKYtNcsz@%j>ae*YT4-}PzIikZ__ zLRKbxU%?!`8T47mK5Oq4K884=cQ!H7J4_}Et#f(V`~b(yL)7P7K&FnEKSM4-cBZmB zSGnAv+>`BmZ<#gx{eD}4RalK~WqB>R9viV4rKr|U)hzs2|6JRZP^Rk})eroL4eE-< z%gQ&Ix~Y8MQNDYXZ?Z+WRUMO`pXAt#`X&GUVKR&8`@_-qv$s1>1$JUL^5yrcIS2Gd z<##{*AdqhCnyyE#FKR?s&N2C8l$01rrT=pv8*=|dON_r+zl>wjPNC_R^~-3szP;t9^!NCt zxpd0arxian=aOD^_9gp&zZ8n;@m|-4(e$xMPIhgG_9&>scyXyiC1C@!dzJOf$Rma`W?#~RnUvLwtBHwW{u0NvlT&jQ&MriUeD&+n#(W#kI1 z!fLF=dTd0@eS7RbSZAHMxT;aEf1GvV^tim%%4gZ<-FGC9yMFg8(dGKDzZ9zFwFb3_ zUlk`l`mJ8NN#w`>%iJ4j?Zj^E#eN(_>lXLFkNb}dI?!pqfb7JV%yWE6{p>M#1Meg(eqfeYnB83*TE>yl2 zE7xn3ua(Nz7Ui7G3g^g%H2YVY$F{M5(X>waL^E0tt^dnNCx;V=z5~>}ko_^){J!Y@ z)hBOXXa0fTPDjV3rHx1X*PIa-t^YemUO<(yd5OG&O8={sPJFs{R`?p~Z>cv>=ePu_ zYvdobZH_}7J%ReEo)xyLCm+?nH~hZ`+17sHE_olV*VVsh*XPRYP^TYKwzhfJ$ffna zX0yK*vcHz91IT~7|6|Y7zVRlV{Q8HV_X~ahn|tte?F-rSvzNj^vWqMtyT4$M0(HpX zC{Dj&DA~N_6Ja=6jPwTMv}9@}JAfRIydPq@>$>c^&b!Wwu2a1~!EcigGr#+`^G~5q zLkX(P>#o!m%%Df#Vy`VU&zL?3(KmVOwI|WH+Uv-9;uc^LmSF2gzO<-5Tqc}-)tD}s zAOALeNYq1{HZRY=>R{tTc?@%{D+pgbGKT(@Mu74=gKilZI%_zloRA47|qxF>W zC$wK=|DM&Sxu;e5hG%VHs7^VM#)O7t1X{OQKNB;w<9lmiOo&7k7!)TVrXpMf1 zp1G?1KEVG@k0W=}`~`GgH-F)dHdDLhLT^Q|QkoW&@FxNm_O8wqL z-bS>q$Fot3{iqUH9tqtwQ-7HP&K1HexeMu^p`&l>aU6Gnr9WbSPud8pP}=<^QAY!js0Dq*wN%C&Ny1H}+yb4&pFkhS`*w#%$2BXzpc5U;`^G609>&!1e z7CAH=|2O;Z38&GF7No8hg`v{PZtW9>lPTwE?sAOyV&P0bHay1CM@8uRYl<9L*t*=Ke%;fa=s02^>E1WcbHT ze;y9K^5^00vrmMBjsGS5c*B1Q2hP3_e$@4!!~Qk@Is9T=v65d+=QrNrhrSOBv z)54y0|22Gn(|-+H_o)vLsSl4B|96g_FX=CTrzjkxcaQMhJNNQ1z3Ug=-*2cdm8YM$ z-$yP#72av%%c;?a_j}n-3qtb|{(@hdkN?X8^}0Pg#CNV!C#h?4?364$F7BMVF!B|& ztBc;aZ2e)MY1T%tS$}Fi&7hJ{F|5S;!)am1(*GL%UcYJQm`Ne&e<}P!+vKo&$dvHC z$&N24hY$w4GS%P|6yaHy)}!%Tbl-keai=iANP|N{qc<7ayS9*iSV|5 z(n)&Pmy5z_GFsbzhCGK0xP&Wce0iubtf3)QbED7yN7?l6cbM(^Ie-#^~@e4bNJo##2v^PKZJ zpY!u*+3(6SWxp@$7=EK{;Jexj{N8v%A6nUS+F13U?v8ygBSNz^K2sM*h1S6%!r@!S zC^#?i&XDl4W7;Lko(Vr+ZLfCo9P96mHvY``{1w{ym-6pk=ij}lT_3IV)T_q&`R@%3 z;5JJBcDwh5yN!N)#exu+Uy&=r~Ywka2P^Qy1%CJ)>ZnEILu z8>Hh+=|I~^=@Qnu!gGqx?Pd2wZ>sytkR%)Oe;FEE{xVE*%_*3M>6nSxsKi{9_E%8v zX_~73Ghh8@q56;a)7q|XfqBkZfJIn>Wmt}tScR^o=4Tiq(L?6mP$yA$$lqdrP()h` z|9>a{Up*&%(D+As#(7zFhfei~L|yp@#y_qRXZ_XkPiwf&QE5AnEn_%?bicHlp%A2csjj;v7McvZQA2KK#CSZO@o zV)c(^>*)Ib9_&M0|Na1Z2nqDc$1#t`ydC%a`qTEnj@{}Tcs$;yjoyx{5lAarMv&^;u~j_~Wx-fpZpN36`Pe7xJxlxSYNctI+rSb72kH z|115+E&9JtyT(QR-)F^nYpk~Qv7vR&*pQlGZTa7au-3WjupS$+302sJuCx08FYEum zssI0~{{KGx|0tk{HuLq{7mp1`kw(WY{r||K6Ak+F8+W?Cv?Sx224sul)^69&j19G} zu>-rX2m5dUhmb&*|8%$L=dT+Za`Bm}+r?*FTHi&TuxGo;J2ZH{Xgph4biWY3#ct6y(s*I$eEYu_kRsMgkbN?d1f4i|6< zSJ00?&2PQub^WOgkd2BO3i)_2f|G&xqzsLW- z#s9x3Zz3yyc4oBwKVzK<@$`Sv9v9@+y5Pn`m`i_)EcJix!%=H)6fKUj0Y&73o!o>)%W#x1kn05N&8i_%3>` zk!?B5N2c#Xkqs#zU&q$4Eqxuve+lpZW_dV7ZjtV-eA~B1`Ys)p^8c5Vv-BiVNMqpX z^3X{p#GfOJIF6HuYxvg*KShsg5;nP?GxT#vjp6^IWuo?nseET)aopi0@(Qw>jin~9 zA)~*chyNeP{oQapfVg*1k*zI_1JuUQFh~3QR@Y!3lY7P4Ep9v>|99Je?&3ZkV$c@v zrCS=+f3}caopY?w$3tD!||43oF|IEZ}RAMgXVF4E5gX>?bYj2P@)otS*VMT3# z|JnN2OT@7Z%aM3sE(}?B-7^?tuNC?ltVPo-<&$#Z@w&mSZ@NEueVyOdV~nGr7c!i6>6~qh2K6Mc9DBf^CkNlJgxunnEZcQIdMY#%7W&1q~Cp%#us@``<#0K zhmgP#B#}blQ%{Gk6Y@VF>&f`m=n zCHp+kOZ)F_<^R%GBK0=^*KrFO*XoP*FFixQ>3jAstIxmsU$i+oH)9My&spsPD3zoC z63b=n0;{zX@cr{|TK`Xfe&bDj{QB~fXy!||pcN^s5zktz!+LDQCRAY?y7-~peV!3= z=)LOMT=$Ih?HBpyZHtXRz){3;0v!+dtj@{aH2#20XrrhV&kpRu9@Mv+|IfGGM^B2Q z2?ywhkQ(paX1T{X(ytvLAuO)@a3r3K>^)^f3`6Fa@3K+;S3Yz)&Y`sK!(REn#dlgK z|6jE}gEpR6M~(CJO6_ZNeE&1#uNB6K_|I{i#3{VLefNyxuBq0)cW#fonM3b+<4Z0Z zUqTk;tG3DPzu%9FGfj3lKQoelFvj>a@`5<3f1=HvOl(z8ChOjo|J$|M(`!Cnu06&0 zfIj#4!2Mm7KX1A}GR`B6{d@)em)pPlpe*+LUGtwbd4n84L0MGlf7BNd%cVFMInGJG z?YFzQkB6u(H;!(idc<3m#+-8fbnSh0z#?3Fz04HHqwN zR_>Am?B6tUI!gZYRr&9x{@ZxYm$h+x&N!gYXt(?3N5X7jl_>mx%_kp^|6f2~ge4gG zqp|DIB`T%_$if844>+1iAZRO{reXj8J z*oaN2!Zy^RfxT*EubSAzB!9HIkL|h123_T!p5WiJeeEx6U+1@`+1d^=+TCb}JI5IR z&-Qk+w>@NT4!f)EJwKA)J)9jzVz_-~U1t~eU?1uivj6jq3!o=f`sp_m3zbP47gh|KYp(2D@7(A&0o9YF>VP;(y2Iv-3Uwd$sa^vv)(rF&dp@da5>j z@eJTL?jpNb9buyN2k1G^p@*G+NFTJ#y&o2Lw`;UV8?NpkETK*?l&nDgNb3)fqflCZ zV2*SRGfvF$IJB}ksjKR_&TF~jzR!Cvm%XQB-V+Khl!qdEZ+TBR+Ty)tjN>5N>%7;4 z-s>IjnV#9}-MpsV0*}`*YNe;pj23L^wpIaO_$}>mu`j3QZ~s^p+Y1VxmPg!A!)o?- z1N*yK{vw-Sm%lUG_w)Mc$SKa5hUu7z*{H-^%tP1W_xd08CFIaMTs|Ko{*m(iQ|cdl zkoWKRQ`!&cjsMmExIi3>5X+1uWaj3eu#8-e>>TwGaussVD&xwOZBHrV$YNd@N3QYP zT2%k?L0RD!e=J)^UyqF#V23u5vH!n{?6-c!HZtb@t=NKS|NFGdm&$-2=mYTk4(!4{ z)NJv*e)pj40R0dWID!Fn!vXzDi5c279jA~+C+hEL=bPwx(3599TXmWu{WxNssnv1I zu}5{CJ;F}n6wYAa=MT!xkr!|Y{oj00c7^Qw+#k#O$!oZQ0o+FV>bqs{Z~wR}EGsN? z=iRdV^xUft%X%)pTlSD%{jc81YY)o`!txs)mi4~*u&m+1yJd|N-z{sJ`fgd0Y@YLO zSqs_fI7Jqo*Ctr)IebU`S6jdk`cPD01iH39EbB(k{)c5b^u8_bjBB(%EGxD=ENffw zZdv=)hh;}^J}gW3JuK^>XXx3B56e1{sC!s8N<5=64iiu>tWF%0=*jW!O}ta+(~z3% z-h{QxaIdr8Et@VZef{0CndEF__r6TNUC01b#)}rg8@&`RPl|OyrMIHqd(e{G$yU(L# z@0O*}A#1dK~}vplqFZ3SZFAN^U~^*#~8H!mH@X2M@}c?mQ^lMz2LmJ-n5j zZ#n;k)?Om4GPD_aT2F+2Ip`A1HaRLNA~GIyF&JVP2GarqTak!A8s5IbHj1^ zo-u@E%~$pN{GGmZdUl064(`%(^OYNXmHYIEC@ND5$a{{xi`6%_yT)gI=hph{|2J!f zI39`$jKC<2#yCva267UnVBlZ%(1M6jjTjHeJ(i8)~rwyU?|P|Gn8Y$lPi5i}UOjdsZNej@#JYc5xp)$j>56>))}P(Z+Tn zF=mMVoZpx25!XH(K>f?c|IZi_4$+f~*|_=YDfA;KjX&JWw!Os03IE5J@0A^@yI1y; z-4DtRF1%Ov;8M$zE^LR{owUGWqW7bDf@okb=UeIW#3!+ z`?B3Du9vm9j|pvcV?z4vXT#B#%&jT=MCiElY)HmyA&s{h<*~zqN@LA_rr#kk_%A}- zlkRZd9K;jyv2zmqxSz4>b=F+``Q>5Z$l0ety?&|2vX7bL{Na#1@v+c!dZfC|@X)NU zD5cM+b;EGu5C+P&O&%!AiKB>`zB^^L!~aKF^{oF^wteS+EBpI5|3}%5zFTE+{kfBV z|A%`I%62JJzB}i?m+gG@zn6V~{<~#+4!+B#KOJhm_H_7R`~NK4cW-d`!7Kl>Y%hMS z{o=qe^(g;6gLAlmpQs~UA{$!b8i4v3PV0ZVtWSY#d7zKsj((Lf)(vz{Kd#{h25=j9 zaUWe9^wVtC|MZl88}#~5zD)lUigWbSyrG{4N0COyV*NPCqI03Jm&?OL@eHcbeu$yS zZY~cQ;T7~;rd-=jc^E+-h2q|F?T_VrCv_x@78due8%It+{VZ*S2i7%Yw8#j|J{N(84rm(|*cVQ3qA@;G9%BQ?FaU6G@*8hK= zpN|}0zL(FR->d(BH$NY3^ZDO6`U?LW9rO&c&g;BW5fZOgghS$qYvUaulc?ACUMD;i zk9TStK&k%mHFaltD_YQuv~$wYmnUTv_|9Bm; zZHDw~D>(X;^rM6Ak9$L8(RrTzZyXkGh-Uz|aTlfW|8>K{eR?vs4HzHrkUpr^`ybD4 zuw75a{}1y%wDm@NIK=NmQGvL3e9gDymEV_#QI1Dr9425AreGRM^D^9HX&uUm-i!2i z9<)y)nQ`2c_g+2ct~&3TUYP2=j`W_rh8IGRT{&7P4$7OW`GV8dL?E7c_|3~%Z|AGJiv~~W)xen{G5t~qj z!Y9(+HUXsm|tJ;>};_jp^o3~Czu|F_=93ci4J z#l73&7|Mk5a)fiYVIK~l{uTZ2%Ev?Wq%yJz3HlMFwkn(VE2DQStI4FW*oTmc5?OjD znM3SD=wZ|2y1Yfl$5C)xbljU!2Ot}+^1p9tUm%kY*njd#`@#zSt^RWgXK)S|a0yq? zk0oYioB7ALg~xf1rTGQMVq3>d%@pp~~eMcPh z;#A+s3%(PywP+tcp?&z6wg7a{GpEIKOa4Sxnf#3&3={w<_p{y#rBOmN;LOu;nN59UA057X(%SLF@g?@an^q+XLZ zggsvWr&8El%)5tDh58SF zey!tm$g)%G$&GkCUMHW?ZlzvdB;y_joBY-n{cE!S?|kQ9*PqUQZgX6V9oU6E*oOl+ zgaih@ssB$NXp|S4q$#Gg`K+{|6>aOp@`$?dd>Sm&r{wHp3MKF7YBR4 znfJ#3k#ViSjyJtuWa*u+YXAQw-~Z?8{$1JuzV)F;?O8J()xS@Q{}j%k@Y^xr0(l7o zU-m42XM89<_I>t~*Ki~LhByW~?tOBb{&@aE&F4KA{p~65KehpJ??Ur<>EEvZd$)dK zB=tu(>z8ghumAh3e&Q46w~OO0?&Bc_y{(>vp{PLDtNP#3gB*I1M*&5&>FaKHjiWpD zb(03T$amX%pjy~!M^qfBG9(~@E=u=R1TzEzQw?6G&a+ z$HsnO{hztwtg*I5?KiBgLSKOYbp4-~{}25?_vCr^w@5rounfzQm9C8Rt)%DP_HAvo z4*-1)ijE78;~c-<-P(19twUmieH_TTXUB$(2(<9^3`C*vLiac*&( zW4z68wb+4O*n{e4tnE)8z#$}X1WBaOi5v=Ff8>*t=D)mYu1hF?wEx{VACTb>b-@Zs+T)1PWBxL%g;~`o~8e9mi&*Vo$~*7`Jer4AzKlB zfJGSij``o}Pkmn;8 zSeq(AHeqUzeU*ZQh#yY(p(} z;QjqSyBv32eQ$l>d+a=Vue0ljV+4x)?Y1%OuXB#xWPizy2kfstge*FdzUbUvYabHV zJ{&;Sd2#={L-gEubqME_#{YfOcQw)Zvs~i^*CG?nNx0S#GKn~LAVsE8+W*dPaSVN@ z;~ZKy`~6kvBx4_X!%g!K)Ki+omqhb!;b=t)MgKXDlQ@MlIEM?kgs$zgRu`R2SbOFDClzjX2VNI>KUFJqzo2M_GXEMeTiPdzL-MQFXI4dSL*laaG0B?W#d!zzAE(#vH~w$8af|dMf4m8M==%`+RAXA> zn8#)u5OxR&96=H(q*2=c%NhuM-}S9Hj{E=Q$Rdv8B+}||r^qunhYKjp|J$#9h5mT{ z-xc~bJXt^Vpt1@LgZV}L!lp6&&yoBhK4%MB`N%1(_n)^;^YQu0TlmcVj&GpG+{W6r z_`LOhqvruX_YVJ_FPvY>Pg~4STgds;`CtR$yN$cJkNUIv zzy0=*9@_+(FzCC=e55YBH?$zG)i+dF+I>}!Baq!{JU=-anf>pr|G(N;e#aB=r2l_2 z|8axO6RS@8dR5S3wqm{AN0rT->st;KRGL` zC!4m*k2ig@_k6o}y#8JPm&`T!1>cpe+UUGZsKPeXVh46%5B6c;lj^IVG@gOp_cxw> zMxKAo9Fa14{wZ;jag0yu1=pa*b%72!H-RHaB84yH=cMP_-CLvgy;J@{Y5gC^`NQs& zJno#K!kb=Z|Ja9SvSlG#L8jP^3croOC?w8W3xSMl)77(?5n&Ayu!-$|?QB1LX7Mq}$~-^U5{>o5DRzNwv3Jhjcf6@61( z#&dLElz!tp;#iN~-NtrEcVVyezb*X>eSb@Re=B@{NusZs+nv#QMxw!7(yfBTBNDGG0JG!ZXZBlP1?JXxZ=P4xP}`Tz-<_(AA0b3 zAHd>e&j@i2UN6$!_Wf%Qq5ZID(IU+&q#YU0rvrDzaUTyc=pT%~ox$!8)0Rnpi(Y7+ z6Nb_&u>GrZ!YFbyYCiMXQ2Ptziu(RI$G>)u6Ua%(3d>*$JvUkYo2Z|TJ{`p|?hpC# z?$2-ipZ^)unnhR=kb9Q$-Qn3)xRTsOm-P3C^R z`Khq~jTgcX$4?FWw!aX5@cQ#%@9O8n_uu?v*i$z(d~fk5!ftx|u#bhd!5<50^OBA( z{g}3qXF^BjnXtpPc3}_p;Q$WdXJ4sVQzdjodzoV`zJodr-OrEbBQ*M38 z@^IwdheN%-u*R~dL&NL(zfSANK+D;WKI;EEKRmQv{HS#sCWdWOCx#>9NTTN0^PzV5 z)KEQUa@am{a`^knQ|&ALd`S5%jei*Psjy30zkBghVdt0^!uMaD5%!#(5jy>r!w+Ab z7544_O!&d!&xF03J gad=E^WeARIEhpEiL&Ypc@7tF30IKt9rlyga03myU(`P1 z9o&2|B>Oybv|RTN&c3L9%KZMt=J(?$(&%{eMbFUu{@2a#_ujh3TOR;Dv#bw*-pSSn z7-@Zg;noKj_F~wguDTU*{_KG374+5jwap24>HVLxZoy~tUDF?8;188y&@OfaH9smZ z?YEScw(1X;hoQnMFap`9<`{n_uh4UM>}`+H^l>O&wzoar(9is(J2 zZh)S9>Izuow4Ie$4;qSjh5H9sj2 zUCO?0^vqCRD$_nN{_!1k5BJoLqc15d&@oHditKQ85Udeb|EFIJYstRm7sEPo;9J@~ z$&J{Axc_(++4n8^kR14(yiV@GF6=>qud|P=+h9Kw@(}9j3GxV%SCz$RxvYML$Nm2e z{LDtSE{?HJ&S94(vP&!2|E27ddcdmqU!;&mCvqs_I7JH`qwugeq)9_MU6pQs1QK_~$*?LEnYq ztM2Vh_oxmM#|rHcwhsr8m})*VS*K22FFz;fN03C5yqd(Hu8+|FOKX&W)cSqa>WTC8 zw!9%dh~;U@|7w1rUEqtuLMJ_k0rxx*^i9!^<0MYu49?*KF5wCWeje9<92WYv=pS(0 z|55A6k~iXSxQ+B3{Y~V3Jj9^g(yz`J>mNhtxy8Os`cQfWp7?_=`BsHL@qg#M=l^cy z|L)}fZs-4I_!r_F;eVqr8t-ra8|S#|ApgIG|69lZ=kNFO{qrmM|MU6(FY*6R^MB9t zfBCbe{qMw?$@BloPX9}gF%M4=*Cb3q{V@LTQy&S_=*i*yUwUc&i!oTKH{IJC?y-#j zD}1_hW@0wdnTk+JX52&8JUCldGc`HE*9 z)?*{;hfDvn#!}FeGu#`h=-ZGQ?B3AgoaTui`G9?@_4^L&LbW$AZI=I!$z#Z&WM|)z z$K<~v+P1TQFU$W6RW-_ibIOO}{{T1J;TwlEQ zzb)_Z#~(au?=O7R{Bpk+K3N_r$PpNYfsb12l^lnf->L`MvtR;!68g=7nL_scwjxX; z2R>$u0yz`2QHi;jhXq)KC0K^#Scz3wgV_GJz5KrzqvsoO9M=datslcq#`0yIu=ESs z`N`^EnbSvBAv;+egYUJC9@iv^Ye3Y}cc8TXmo(;&$^T@-b>pwlbWdMCn&~ZQWgAi$ z{Q1>w(S=C zhfXBkFwQ_c2^>KZ^{*L!@T&1&@i^LMr0Ja~?Mo!ymaS|UnG;q-TA%!JGDDsuPa%7C zNZ8|@pP}dS<{%t4Ho(}uUhNA-vVev%_J5*ljAtj&jFzeFzp%K@@j3r}fBVJ-$Cq#g zg-`LnJ)>@v_D6h2`aO$03iKk{#MSYp9Tl(;uw7M;qPxc2lk@l3}|%trm2-oNjnlAb&$ z4`VKU9-fRp^4-V&tGJHR0^wU$v%61O^P67ykuh-Oa#Vl97&x-_tHvXJ_qni&z6NWt z4(qWIv2U(4{@|>>GRIZOy1#8?Epo@C@s6|&mv1gg8(Da^EbQ>xF6_ZR9KazYFrZKW z2-(L^NRs{5@k^0ubRvgR`;IoAn4gP|3txIJRDa3Z{@QK_j0x`3#?$vl_W!fi^YQ=A2@IPfI0j*ve%sa`~%|$ zt{XpqwmR$2w_AT6>7CZ^-)sK<9rO2{(@8ey?`}lXRQ>n*(wishzo)m7DRDk@kAr?7 zKVT>-&@Z2iAV(p6bzB%tj>7~@LiVO>U2D{i-?y z*@+yA$g0=JH3pBzF!hbt7kH9>D*oQ**~oACTaV_yCFJ`v!q4FXE}_0#{%MzI=#R^v ze)=`UH38z}t<%vqHjMMz1WdvdOv7|kSC@yl zzu-)IM*CuRWLcO^uf$wD89#PM+8=lZWbvjpSGJ}x&&FifzjxRawyOoL^wcq9jQw{3 z7GVjNVL4V}6`t(BJIweyZT7v$qktmX-e&)B^mT3RtF^yx*8War5%=Fscut8~<_d^w z1nO64k0;k5?g7*^-+f`dk3yl`=9ZPkN1sjepWisa9RETCL6&X|JnT7IfKGo@!ZEl z6u$Cw81zGBDTbl~UF!PX=((u=e^#CRp1L^-`_=#Vvj6J%?dtkRV_jc9>>x9a;~1Sz z=WSi84Pd_CRKeblzl^R~G81F_y;@&g=4pMA7*$6*2{A``+vj6HuK+q;u&J5bE>TkFq^EzT+G7)EW*HZ`dP_k zQD>jySc}K&ImYz|mJ9FKH?xwA>u;=y=Y3sY%|B@e`1!D~*6}*5$42z1d*#qoH#l^g z`%qB#Dx#O3S2sH$WC@o(D?^M8>>2QrQP-`)IQr0Hq#-1dESk~w5soP%OKUd^|5 zU&rYuQGA_mEiAu*uZ@`R6F2!qvHnM|zy6VM%6Vr{M{m+z5c|;2Ilh2adfdn0@jU;A zSEXM(O`DxZHjBUIj`Tlp9$91DS*`bY$$zh)AJ=dL1NgxDkL+sfbIPBP*HFs;JEecG z^dFQ?>HV|)cU|MQc<$mp9wNIz`mY-Euuu9IOaJ_jgdy~yDDD^cZr6BO`h`~rOSD^u zfgFYU7d{e3ljBh5SvL)nu95afbUX>I=k+VP$7bQN4W&WeZ=5QROk`8#_hz!iaVwdU z&!+g#G)%`#%tj>&-_XWRcI}k^`Qkmu)ybQAd2|E&vYCA$+pf$1I4ZBFZ^{1`<#A-Y z<^PPl-{QSx*vGuS$ivnW7iWfyYxymr$NdAAkUitYk6z>_D_@2y1CY3B?*A{!!}gAK zVVU!mqh44YR??HNOB-s~*V@>Aqzr0B%W7#AzREfM!5EY;ea%`n6=99zwOEJsINUfW z{Pcl&BF6@WpAAzF(FQ%hx9?XbZS?ylRH3@OJQT2v9{2pJC3j#K`qfwVkbPfN#~}~k z5E6KQ|H2W+Nu;oA_vkS26Z1cQt8Dsbc|}P@z3^PvHuAaf zlg9FJ%JCVT!v$QzkI$A{r>#8v!1^A0uYNfE5c@J84p;ovkMF;t{K4VfAzX8uDEn6U z*?#?Z-QNm7-~Fv{WdAoqec$HLxb@A@aQK@c`Sv$LQ`t9svtJL*uY4n@*;W4^{W_-g3D4RH?OHtym+9%9gs*e?u41sdiUf1n+p2}$h&&6m|H zP8ffH)EMpV&UwH8Kg#iFj6*?xd>0$qjh@NcBD5Lwj_3cQfFjzI!Ro3)PD9jd?cx6ba zBcT4|7=N9p8wBJY@Vq7Z?X12q_E6? zmSZJWq58Ada3SBPUH zHX%K0aHt|P>f~AVl5O-_>_E>w@uQa?o=2QRTU^br@LT;>_Wpq}%k;!*HuOv13N^p^ zR;c~Sx3pnN%d67lT5)V|T*D*o^?ks(hwz`R|GRa&HbEqu`_@f92$@6*X?&pk*Ut8z zuK%mf_x}1{r#Nyb;yC&~r9Sc%a{@l8KJrWTwvTTPCxs8_yBc^#pBViN&fx;mD?E$O ze=A&~UqL_OxTd%k_%(V?`s3OxH|PV1dy*HBcTO)72RDa3!f)d)?xX(v=1`Z}93Ik> z_cr^6Hitp`8dcoIioPawMLWUqK-+Jf`-?>9U51BJ2Wl*c1Uio3e8)qLJL}vLfcDK zp&dt&M#qLKb;w88?(6n0dVY-Sk92)LP~t&VnC3duF%z>S@h!jti5&#I?C#ZIHtZcuJ)opU##3foYN9q8i2b#FHYW~cE#WbZ-# z9a(T}K5uAy)A%18bzb_aF)w_-44Fmed11bvUE&Ze&3J0a)=)DbPp2rBPg=J zg;{Ly3v4kNee=Y#@|E9GsJ|?qlbxu$D6ie~8*+||Xx+>PN83xbZ1DeoDi6nnpG4u8 zgTpEE49?*K-v9hAIlh8wmRfSc~>;KHRwLg}Jw=T-(_5|2c zr(Q%~6My@KZ-?yrXXALt^%$uBcGyTZDRYv(olW#Al-74SsD7vZ8P|8&CM>bkn$F}7 z)GzeD$vvohS^J7-w1mD72XF`p96=Hd`}q}Uy2*|_U^lMvE9j;5e=f6cY*2d(`$opG z>>c!IyGr}}EVQo72IKQKYkMbi^R*AKRe2QXaeiLCFlr~cTJB2!W(_r^K%=iJv=g>+|-C^fP zN{8P|^W&B}56QQsuU&Xv`Wl^wdFo|)foi{~!x;~`?*ATvul{sHNKP5Pg;&JTSkiYwTE5ku!oM3MKwplPh+}zr_!R{e z=~a%4u2n!Dz2Z+lWsHsRwOEJs$X<32*hp{ijvG;0->Tblp_kf!%!^Hp)Tid0Dr`e7 zc3>CwU>~}k<+o3@{=mdXethr6M}B+ZG(Y|Xzy0;Wp?w2C9_g3)?JM~0bF4owgC9S6 zaOg89vhR!R{m0n*Uh_9R>n)8T91!nzn5?8I$5&Q zb<_ONYvQ?q0o+EM8<4?WdQLxh&+w0g`}FB3zQX_K6PNsd{_8_w34ZXPpD5>1PajHF zppM?ORJvC1qkcXpj1bm3Qo6=SC;vIF^ViTW{b)Mvyc5FRYYW+m=>M1If8Ej_j8Xm{ z$K2L-SZ|X)4o}v9RNm&~`CjF5{sH^GU;f`K|L>Ikm)KvV(SZ!&JmF5H-Nyv+Ou`gQ zLsnSEeNU(7)IFpBKa)NiMaKo?#odcaVF}M+E;$eN^abQ1)D3^n|39Y9%JDL^(&OB} z=CkYrO5@+=@R8;Bq&(cbU;alcJtYq>_n(zmg*8}DRvxdYYB>=jx2J7vYw z*+(P98I-`!_MCHS*_v$awT@-d4Cvge`U#bXa z$aA=W{_lMxTq66dyLlxZ;~H*Y09&+iY?bD?7f1CogYB`Q%~W{dW7f4N`+q$sJS6+H zLk~LW`}wtc%9pHdPald3jKC<2#yCvCBn&hg|Nr}P`+0mMOmQ6N|4k#OVaK7|6*k)J+5icgeCN4NKKUgXUYG3g628$|Ni&vdkZ^}SN^f%E%a70wtwGt z@3C*Dv_JN-Az`^VRwB;*Tt#-VdGsRoD;& zlK;EYH?o)g@Agfo50u9Lkg3yg{-1v4uUHE!#`)zz;R@N0Yq)`~4f6kHd5p}-=e-x@ zEfna*W9)yP|1bWd+xZ3ZVaHN=bA|j*b~=vZe+I-;{b6fzl6UcV{Lg**1 zH6Pa>`lT_U=N|jOhLrk;E-J&@*#`{#G=$~OTZvUzgSA+PxZg?j*PaV~ztaEZIL`m; zH;#29z3)@%{A5C$RpfT#vbK@6sK4&r_Zgc&Pr9e34fc7V??Gw~-x4iL*#I=$)c>!) zylIyHXEaaNFOAlN`k!}O_uqf^;Q$UHfw%5Sv;Na953GOPsQ({5^Y!1O_Z9v3FYCWw zsQ;d9yJi0WJ@fyMnd^^^ZvFqrw&?$_Gd^Ir{(T%pS{r2tGTJP&EA;O#9v8aaFh1Z- z;{#q2KYI7FF9+Ed(wvlV#QmJ}Z6wJQ(&$9Zzq28~*1xIEv(dPMrnB~^&)c8A#oh(t zY()xbY0f#nh~qektaN8yHdmdV%Sao}(9fZ`Q~&=~{r}GCb^jNHB{q)@m&hxqKd+yb zyoS0m&sF-9xZ!vJttUL!cRc4l&)K;R-a+G5^ITy4UhNy^zPxRma=Y=Pxa~i8aUTyc z=#Y92hN1#pY-~4M+e5~=Fui2nae*xQzS|C)`_f_#Or7tuoBywFneqIxo?n~`GeR66 zXy0@^8skv-n0@5P>IQvjyXP4_s&>Ju0q^Hqe&e;c~hICbp%(9NKiJ`T7xtn0)5gh@>8bV?B5RF1sQH3*Zs=KI862VKqHp&mADf;+ zF)!~T-z_gM=D&Z_Hzd6O*S^AXXHYmpo{M_(xNw2Ige&OB zHQc}eZb$hyW5Zpt?_b7-`(*!fW5Yvo$WOh0R3NUs5c|FoSJhFy|2lGn@am71hkAM( zyEw{ma;tJcdDHTSe7|2EaD#FH4ddBAcBE+{J28h{AWP%l*oZj(e~;@8^8Xs;bM1HS z&q*JL37CW~wyGO{X8(4wf7{tVaeiR@KO2}q_9pvx)$gax&pU5^-dSw{H;oVAn`a(q zKe(=bfG+Lv-4oR>rfNSROY8QN1;<5lihGzA@5g;kCud?dDlr!gBc+ecY!Y`;yv<~b zI9pva)$N*tjd5_^0xZH3l>Beu%jj`WogOxGIejIHudu(1-}Ap$n;#)8@sc_Wxfap? zUPo5*5lZ9lJ=?hMPTXH?qwrSu6YC(QafalR^%tBI$LTk3@ciq%n-=eg+;Ux6i?<%g z&!5m9_$lj!|EsYM;;KT|3HkrD{7>dC%m4Hz<2w$je_z)AHO1{>4SsA4&l472XXvgcChxX%Y(x{$E9_9+4ckULulQq-hk3N z8oRX{3QO>bkB~{!zvOo^jk^7V!&|%m->?mxj&rE#{;N=XeU$yT{wi$W|5xGf5C502 z14X|b$3G}{cfB$;e0Tqduygp>@clRbI_$an*Wsk!27dR6@WWdl3j4gf9|+s)z5jT! zdy}41em{eA==s_lKv;9_yxp-^8 zH0%!MhOwXe?-~zU18%Q5+7tQ5bNI*n%2uS%wwZs+S3f$KFF#fP_AI_DO6#~e|H=9Q z$MjvIx0~;NkPp9C|32S1J;(b0uDJkCpE2 zr=(*>Sm&Je=qUT7cACjyBYhL9uno1?fnC^xpS>K{(6Ubi{qVxFP_NzRNZDt?&&$d~ z-Ddr2WuFbnm*<3r`Ljaf)mh5i7eZ?D3!!<1xol;hQvcHa;GW|gha{Sxbzf*5=02yI zCokSZNZ<&PNFj|*bdAye;yL#?H%Io)@oePz*vC^mt^HxC_p?Ge$GjqgJtK~Ds5|ewK~mTS$CuEW8Kztt7MlI1C2#*n_5a3~`5!CP)0ZkY-c&C~sr}(? z{)Yeb;~H)t?p-rL-o{;YIj4KII>zh#5Ax6UALfI!<0#VTKn7WK;=VW@V$jdzD-1;( zw^Kp(Ti2lQ)$%ZcJ_>Q{&uDTSCSVfMdySDGr(q^$BaUszw5u1()xQp*Ct3W z;PL*&aX-ku=dJ(wVRQWT^Owd$oK=2cuK&-&0xZH3EW>gPe8-rD=e5s`R9|wu{g+e1 zDsm0#XBqz>f3Kw{Pk6RiM_-Q=y%jCW_vq_23}YYJq*6QaQuc@aYC&r^JB8`~vk{w6 zg>9(C4(vkLLiTwn`@EQaevN(R)8|*S&#$x3WZOIJKaO7J=bq>Po@W0yu%1IU(_)j!(ZAC8`Of5+S(GViFLxPF|U&^1{5{xEewGB-w@kIXwR zkVSRCPI2Z?#BrR&si^y=&X5gLJ>MCgFJeD@^I6wLD^iH<|L2@{0he$E*%#cyEM-1D zH&Xg>jeY~g*VQG|JMtT(|4s7(gvIv%+vHu;3%^f3M5+CMvFl<`!uO5V@vb}Beavwm zuJdQ@|BKbD(TWs?`cDN4zqi&8IU3_I0h92bw*S8%|KaEi`A>cQPuu^u$hTYhCU5ak z;uydw;>?_vKlv`x=yCkdbn@}M-Cn*x?0YPrh+6km^Oetrqv|Cyoi`i(A1)7-WZ&;U z6y}lxK^>1=fJIn>Wmt}tScNrMi~gVTC&<1A{j=o2znL37#8QNORPtTa2-fn*SPV>|Eo1c#C+vcYu;kP*d<+ix);y&sJzo_l%#qf}x ze8auH_Tr=b&!2j-{=h5lb@hv3(9b=C-IZYoITTsft{_Js(_N`u%6)CF45J*6M!`MB z`zqb%Qunvl{T+0FjqcAiI}W=)WYLMPtI~H<`pDb^^XKXL%jVCYH-FwU94F2Rn1m^q zh7s<2Iyn;!D?H!v?|Z%@Jzw#(-tm0Jl@QNt=T%}Z=AnL!^gCw(JvqyNUwAPrqAx*e zu>YcExb!ptU3BLKj2sD>hZtk|6L-lpdZ(81J%!~bC9=j7jgdIeexmV`d@?U zl&u(w3Oru_YXtrA`d_2yqmfbuw8nA(vHg9VunCxiDVT=on2CW;`ZmbE&%PKc$^Orj zg)NhnMI)7$-upaZasFQ%LlMUgEO5LCkLUj_p+BDgw~W3V|9|KIos-77{#UAh8_Tnj zz6xuw7XRD%f4kWz6!CvE|8Jdm)?*_!q5ic;^Z%;okLUkwqu1h3=l|^xmezK#i`;|k zts!9_c>o!HU60@5{x*jkCs5$?7twn{J%#LgP5T9U-qwDPUbZxkIL59te{s0_!5H-e zHaP9P4mLM~EWPtp_H#b_zVvS|J=X;ZX@RZyX1X5#FP1deEtv2|7$ls5C1pw{|42|KNyM%)W7*?{@)1t z{eK)1rjs*K^BexR zwe|k9@qe?OTZy@thXq)KC0K?JjQ@M*eeL`7;xYbLM){Xl{&_xW{OSCU<>H8K2P?@{ zC_MLcSVIn2UvJ>2@A?0KV|>8Z)dzoMZj9g7VLdh?j^EluR$&`z@p$~-4*KKqf4k`W z@aNY_Z1vR_pA&b=X??bsq^w z{3mmme`&6LlAb~uJue#{U_5d!nYT_rfh-zV-M}wxL1a?R16q#R~q1`y)FXXPlRPL)=JQ zx1XYTF5wFLQO`%NyJ@X8`s4KnZqNsi(l*h`Pk+4rz-?jg&);_)-^WA5`S)?&&7dRR z%Wt*au`@&HLs5aW{L}Ed^uMibR-0^cx3*c|Op9-)^`7=WRR2^Py>mxlG{#{9CSeM? z{87(0&Tj#r|Up2FjGwqu=TmGH;)$~E-0Sci>>>;G*ctFR5V*nwTx zgMr`KCy^}mKm4sWefpsozD3(3+2MEtHkiH4hw0r;SKE%XhO2leRZ?V zXr-r6-E5qW^KvNSI8Nde&Y>uo0s{i22<`P(+AXId&#`x0$J2H zQCk0h2K&t}#<_MK(p8#&FKwM=asIva|JA3qX4I{)NSyVDt^Y542|cMk5!e5({;9F> zj#IJkK%JsR{LT6&mJ5sf3YFIXf5v;w*bhnADrEA~KA%lrs{ZYGEef+dcl5sOJ&+CV zt5Kbyv`)`rHvJ8@9j)E+KVtc@&i~e9BQ~K5+fa+HM&BRe7}Fej-xMzj^dd_0f6vSR zNH_Za#2NR7&(b>&tACu;zAv6#*n@qjzs3IE8ypVMh0v{MZR%U(tcuovQ z8l~@+KhjB$-*JvCqVNfGaLH3RgPPwMSD^lVj(!1`5a-|bf82gs^uB+xHXb?fDdX75 z8&St?+{Jx7L~MT>)ad)fP*mV?``ZY5ZZY3vzP1bcXv8t!ao?N#OZ*42Vd{U-{>Wyh z-%q| zr}T4A_n);`hxOQqO{l^)biJ+rA3gi^qwAaL)wiEVVYB}K4f_A}cekI`|4*hb>;ES+ zj{VaZ!1AIWIUhk z;@K%4ve~m~@myLTh{wIR32)ykeMr|yU*5gH@BNm(kHx+Z#I>0IwERs^l%5YOyIiXn zuZ!$>am=u0K0Wt_^uJ~-0{sl)n8GLfAG~UR24M;E0(lAbtH*{bWIyWsx3n&TxUMs1M&R7`^#{fyo>vIh(S&A6$~0jDU%Y2;v9Q%o^M6wwu}DP}Q^S)3w@D5A)*j3w6r z2NY01Aqpts3cX#u->cVqFQ%AciqeX%Xhji42(y@Cgi@U16q)_bHTEXJ?*8%nJkQKI zXU^-qJTuSCA(uY?U*TEgNA}$k?k{~|Xqvk@e0cZG;bU{g%WAF*%|*t`PTmqeQEEJG z=k=jw_x0i9MZXq4>6#wi@T|~k-0q=TZHKm1;gMG3QO!4oPaWqw@i+4Q(_GhG{*Cz) zbKt+uN9KKeLXKYyBLzPR!v#-x)=z}Nf*p`R=ZE*>H|W2K#%(VQ$Bw@+ z96kR0(6H;4aIbs$K;f_Ie~R-53&VXIel2_m9~?CPxcB;Sf4T9;ozDtKzgQTKhO5E@ z6~+wl(TZn<=Iu9x2kU&h>f`52`-P}iRvfM@2oF{1SM)t(zsM#L|J(Q-+=<`C?_=qI zZVZ1wEm-s8B zT~A8!2SpFt1$F^o}6;wkeB3Z-8vedo?1hXG>}gBYqc)_@WEljkS2J6BFRe~Z7v z-=m|B{awtaruQUW8~%y@&xrp7=o5}UlD+s>asP%z{4ZRoAJl#SF1&>QME?K7#+Cky z{!^^(k+;x!g8xs8_WzvrY0Q_)+W&}aOvLr4WBdPC>~}4G6~BgO;YR!h#-@$`&l>;7 zw>DN!rT4hoJWBM zcW@_u7sE5=5tDy_`2V{SFw{>KpFy^d_~{E@~e7L4s5lg?u7t9a?P;REzI{`jF7_gQ0Oada|4!>evVr}Nr_BH3@Bh#CNs(!G_`lK{^%)*}hBg>`dXl|A zrEJcy!?W!19NAgL{?&KTjUMzOi9Yn>-=wjK|Al|Y68;1KiT}b_0s9{lm@Jikk@U-? zUmnx<{kxVy_WlrCe|Qi3zmomG*Y(jyce4Mtm_s0))xGi~egz$y*#F|MrT4J+yY*-M zD*e|GeFx&&fJrj`L+)APZp3fkH?j2GqVRn3h4?KjveSN>TzIB`yOW2DwNu!NQ~&;nbp8zg9e<9r?