Lucene 9 upgrade (#4867)

* lucene 9.10.0

* lucene 9.11.0

* lucene 9.11.1

* lucene 9.12.0

* lucene 9.12.3

* try to make SuggestResultCollector generic

* try to avoid second search/full scan

in searchIndex

Author: tarzanek

opengrok-indexer/src/main/java/org/opengrok/indexer/search/SearchEngine.java

@@ -48,7 +48,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.search.TopScoreDocCollectorManager;
 import org.apache.lucene.util.Version;
 import org.opengrok.indexer.analysis.AbstractAnalyzer;
 import org.opengrok.indexer.analysis.CompatibleAnalyser;
@@ -132,7 +132,7 @@ public class SearchEngine {
     int cachePages = RuntimeEnvironment.getInstance().getCachePages();
     int totalHits = 0;
     private ScoreDoc[] hits;
-    private TopScoreDocCollector collector;
+    private TopScoreDocCollectorManager collectorManager;
     private IndexSearcher searcher;
     boolean allCollected;
     private final ArrayList<SuperIndexSearcher> searcherList = new ArrayList<>();
@@ -205,18 +205,20 @@ private void searchMultiDatabase(List<Project> projectList, boolean paging) thro
     }
 
     private void searchIndex(IndexSearcher searcher, boolean paging) throws IOException {
-        collector = TopScoreDocCollector.create(hitsPerPage * cachePages, Short.MAX_VALUE);
+        collectorManager = new TopScoreDocCollectorManager(hitsPerPage * cachePages, Short.MAX_VALUE);
         Statistics stat = new Statistics();
-        searcher.search(query, collector);
-        totalHits = collector.getTotalHits();
+        hits = searcher.search(query, collectorManager).scoreDocs;
+        totalHits = searcher.count(query);
         stat.report(LOGGER, Level.FINEST, "search via SearchEngine done",
                 "search.latency", new String[]{"category", "engine",
                         "outcome", totalHits > 0 ? "success" : "empty"});
-        if (!paging && totalHits > 0) {
-            collector = TopScoreDocCollector.create(totalHits, Short.MAX_VALUE);
-            searcher.search(query, collector);
+        if (!paging && totalHits > hitsPerPage * cachePages) {
+            collectorManager = new TopScoreDocCollectorManager(totalHits, Short.MAX_VALUE);
+            hits = searcher.search(query, collectorManager).scoreDocs;
+            stat.report(LOGGER, Level.FINEST, "FULL search via SearchEngine done",
+                    "search.latency", new String[]{"category", "engine",
+                            "outcome", totalHits > 0 ? "success" : "empty"});
         }
-        hits = collector.topDocs().scoreDocs;
         StoredFields storedFields = searcher.storedFields();
         for (ScoreDoc hit : hits) {
             int docId = hit.doc;
@@ -412,14 +414,13 @@ public void results(int start, int end, List<Hit> ret) {
         // TODO check if below fits for if end=old hits.length, or it should include it
         if (end > hits.length && !allCollected) {
             //do the requery, we want more than 5 pages
-            collector = TopScoreDocCollector.create(totalHits, Short.MAX_VALUE);
+            collectorManager = new TopScoreDocCollectorManager(totalHits, Short.MAX_VALUE);
             try {
-                searcher.search(query, collector);
+                hits = searcher.search(query, collectorManager).scoreDocs;
             } catch (Exception e) { // this exception should never be hit, since search() will hit this before
                 LOGGER.log(
                         Level.WARNING, SEARCH_EXCEPTION_MSG, e);
             }
-            hits = collector.topDocs().scoreDocs;
             StoredFields storedFields = null;
             try {
                 storedFields = searcher.storedFields();

pom.xml

@@ -60,7 +60,7 @@ Portions Copyright (c) 2018, 2020, Chris Fraire <[email protected]>.
     </scm>
 
     <properties>
-        <lucene.version>9.9.2</lucene.version>
+        <lucene.version>9.12.3</lucene.version>
         <mavenjavadocplugin.version>3.6.0</mavenjavadocplugin.version>
         <java.version>17</java.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

suggester/src/main/java/org/opengrok/suggest/SuggestResultCollector.java

@@ -27,6 +27,7 @@
 import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.CollectorManager;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.Scorable;
@@ -35,6 +36,7 @@
 import org.opengrok.suggest.query.data.BitIntsHolder;
 
 import java.io.IOException;
+import java.util.Collection;
 
 /**
  * Collects Suggester query results.
@@ -62,6 +64,29 @@ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOExcept
         return new SuggesterLeafCollector(context);
     }
 
+    /**
+     * Creates a {@link CollectorManager} that can concurrently collect matching docs in a {@link
+     * BitIntsHolder}.
+     */
+    public static CollectorManager<SuggestResultCollector, BitIntsHolder> createManager(LeafReaderContext leafReaderContext, ComplexQueryData data,
+                                                                                        BitIntsHolder documentIds) {
+        return new CollectorManager<>() {
+            @Override
+            public SuggestResultCollector newCollector() {
+                BitIntsHolder docIds = new BitIntsHolder();
+                return new SuggestResultCollector(leafReaderContext, data, docIds);
+            }
+
+            @Override
+            public BitIntsHolder reduce(Collection<SuggestResultCollector> collectors) {
+                for (SuggestResultCollector collector : collectors) {
+                    documentIds.or(collector.documentIds);
+                }
+                return documentIds;
+            }
+        };
+    }
+
     /**
      * Indicates what features are required from the scorer.
      */

suggester/src/main/java/org/opengrok/suggest/SuggesterSearcher.java

@@ -238,7 +238,7 @@ private ComplexQueryData getComplexQueryData(final Query query, final LeafReader
 
         BitIntsHolder documentIds = new BitIntsHolder();
         try {
-            search(query, new SuggestResultCollector(leafReaderContext, data, documentIds));
+            search(query, SuggestResultCollector.createManager(leafReaderContext, data, documentIds));
         } catch (IOException e) {
             if (Thread.currentThread().isInterrupted()) {
                 interrupted = true;