Update to lucene v9.12.0 (#737)

build.gradle

@@ -35,7 +35,7 @@ def _artifactId = 'server'
 //This is for https://github.com/gradle/gradle/issues/11308
 System.setProperty("org.gradle.internal.publish.checksums.insecure", "True")
 
-def luceneVersion = '9.11.1'
+def luceneVersion = '9.12.0'
 project.ext.slf4jVersion = '2.0.16'
 project.ext.grpcVersion = '1.66.0'
 project.ext.lz4Version = '1.8.0'

clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto

@@ -601,7 +601,6 @@ message VectorIndexingOptions {
  // The number of bits to use for quantizing the vectors. It can have the following values:
  // 4 - half byte
  // 7 - signed byte (default)
- // 8 - unsigned byte
  optional int32 quantized_bits = 6;
  // Whether to compress the vectors, if true, the vectors that are quantized with <= 4 bits will be compressed into
  // a single byte. If false, the vectors will be stored as is. This provides a trade-off of memory usage and speed. default: false

grpc-gateway/luceneserver.swagger.json

@@ -5404,7 +5404,7 @@
  "quantizedBits": {
  "type": "integer",
  "format": "int32",
- "title": "The number of bits to use for quantizing the vectors. It can have the following values:\n 4 - half byte\n 7 - signed byte (default)\n 8 - unsigned byte"
+ "title": "The number of bits to use for quantizing the vectors. It can have the following values:\n 4 - half byte\n 7 - signed byte (default)"
  },
  "quantizedCompress": {
  "type": "boolean",

src/main/java/com/yelp/nrtsearch/server/luceneserver/ServerCodec.java

@@ -23,10 +23,10 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 
 /** Implements per-index {@link Codec}. */
-public class ServerCodec extends Lucene99Codec {
+public class ServerCodec extends Lucene912Codec {
  private final IndexStateManager stateManager;
 
  // nocommit expose compression control

src/main/java/com/yelp/nrtsearch/server/luceneserver/field/ContextSuggestFieldDef.java

@@ -25,7 +25,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.search.suggest.document.Completion99PostingsFormat;
+import org.apache.lucene.search.suggest.document.Completion912PostingsFormat;
 import org.apache.lucene.search.suggest.document.ContextSuggestField;
 
 public class ContextSuggestFieldDef extends IndexableFieldDef {
@@ -116,6 +116,6 @@ public Optional<Analyzer> getSearchAnalyzer() {
 
  @Override
  public PostingsFormat getPostingsFormat() {
- return new Completion99PostingsFormat();
+ return new Completion912PostingsFormat();
  }
 }

src/test/java/com/yelp/nrtsearch/server/luceneserver/ServerCodecTest.java

@@ -32,8 +32,8 @@
 import org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat;
 import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -61,7 +61,7 @@ public void testPostingFormat_default() {
  when(mockFieldDef.getPostingsFormat()).thenReturn(null);
  IndexStateManager mockStateManager = getManager(mockFieldDef);
  ServerCodec serverCodec = new ServerCodec(mockStateManager);
- assertTrue(serverCodec.getPostingsFormatForField("field") instanceof Lucene99PostingsFormat);
+ assertTrue(serverCodec.getPostingsFormatForField("field") instanceof Lucene912PostingsFormat);
  verify(mockFieldDef, times(1)).getPostingsFormat();
  verifyNoMoreInteractions(mockFieldDef);
  }
@@ -97,7 +97,7 @@ public void testPostingFormat_internalField() {
  IndexStateManager mockStateManager = getManager(mockFieldDef);
  ServerCodec serverCodec = new ServerCodec(mockStateManager);
  assertTrue(
- serverCodec.getPostingsFormatForField("internal_field") instanceof Lucene99PostingsFormat);
+ serverCodec.getPostingsFormatForField("internal_field") instanceof Lucene912PostingsFormat);
  verifyNoInteractions(mockFieldDef);
  }
 

src/test/java/com/yelp/nrtsearch/server/luceneserver/field/VectorFieldDefTest.java

@@ -151,11 +151,6 @@ private void indexVectorSearchDocs() throws Exception {
  MultiValuedField.newBuilder()
  .addValue(createVectorString(random, 3, false))
  .build())
- .putFields(
- "quantized_vector_8",
- MultiValuedField.newBuilder()
- .addValue(createVectorString(random, 3, false))
- .build())
  .putFields(
  "filter", MultiValuedField.newBuilder().addValue("term" + j % 10).build())
  .build());
@@ -460,16 +455,6 @@ public void testQuantizedVectorSearch_7() {
  0.001);
  }
 
- @Test
- public void testQuantizedVectorSearch_8() {
- singleVectorQueryAndVerify(
- "quantized_vector_8",
- List.of(0.25f, 0.5f, 0.75f),
- VectorSimilarityFunction.EUCLIDEAN,
- 1.0f,
- 0.001);
- }
-
  @Test
  public void testVectorSearch_boost() {
  singleVectorQueryAndVerify(
@@ -684,7 +669,7 @@ public void testMultipleVectorSearch() {
 
  @Test
  public void testHybridVectorSearch() {
- List<Float> queryVector = List.of(0.25f, 0.5f, 0.75f);
+ List<Float> queryVector = List.of(0.05f, 0.5f, 0.75f);
  String field = "vector_cosine";
  SearchResponse searchResponse =
  getGrpcServer()
@@ -1376,7 +1361,7 @@ public void testInvalidBits() {
  new VectorFieldDef("vector", field);
  fail();
  } catch (IllegalArgumentException e) {
- assertEquals("bits must be one of: 4, 7, 8; bits=9", e.getMessage());
+ assertEquals("bits must be one of: 4, 7; bits=9", e.getMessage());
  }
  }
 

src/test/resources/field/registerFieldsVectorSearch.json

@@ -110,17 +110,6 @@
  "quantized_bits": 7
  }
  },
- {
- "name": "quantized_vector_8",
- "type": "VECTOR",
- "search": true,
- "vectorDimensions": 3,
- "vectorSimilarity": "l2_norm",
- "vectorIndexingOptions": {
- "type": "hnsw_scalar_quantized",
- "quantized_bits": 8
- }
- },
  {
  "name": "filter",
  "type": "ATOM",