Fix Size Calculation (#2052)

- Fix Size() calculation to account for dynamic entries of the struct,
to ensure that accurate information is passed on to the memory throttler

document/field_boolean.go

@@ -43,10 +43,15 @@ type BooleanField struct {
 }
 
 func (b *BooleanField) Size() int {
+ var freqSize int
+ if b.frequencies != nil {
+ freqSize = b.frequencies.Size()
+ }
  return reflectStaticSizeBooleanField + size.SizeOfPtr +
  len(b.name) +
  len(b.arrayPositions)*size.SizeOfUint64 +
- len(b.value)
+ len(b.value) +
+ freqSize
 }
 
 func (b *BooleanField) Name() string {

document/field_composite.go

@@ -68,13 +68,16 @@ func (c *CompositeField) Size() int {
  sizeInBytes := reflectStaticSizeCompositeField + size.SizeOfPtr +
  len(c.name)
 
- for k, _ := range c.includedFields {
+ for k := range c.includedFields {
  sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool
  }
 
- for k, _ := range c.excludedFields {
+ for k := range c.excludedFields {
  sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool
  }
+ if c.compositeFrequencies != nil {
+ sizeInBytes += c.compositeFrequencies.Size()
+ }
 
  return sizeInBytes
 }

document/field_datetime.go

@@ -53,9 +53,15 @@ type DateTimeField struct {
 }
 
 func (n *DateTimeField) Size() int {
+ var freqSize int
+ if n.frequencies != nil {
+ freqSize = n.frequencies.Size()
+ }
  return reflectStaticSizeDateTimeField + size.SizeOfPtr +
  len(n.name) +
- len(n.arrayPositions)*size.SizeOfUint64
+ len(n.arrayPositions)*size.SizeOfUint64 +
+ len(n.value) +
+ freqSize
 }
 
 func (n *DateTimeField) Name() string {

document/field_geopoint.go

@@ -47,9 +47,15 @@ type GeoPointField struct {
 }
 
 func (n *GeoPointField) Size() int {
+ var freqSize int
+ if n.frequencies != nil {
+ freqSize = n.frequencies.Size()
+ }
  return reflectStaticSizeGeoPointField + size.SizeOfPtr +
  len(n.name) +
- len(n.arrayPositions)*size.SizeOfUint64
+ len(n.arrayPositions)*size.SizeOfUint64 +
+ len(n.value) +
+ freqSize
 }
 
 func (n *GeoPointField) Name() string {

document/field_geoshape.go

@@ -48,9 +48,16 @@ type GeoShapeField struct {
 }
 
 func (n *GeoShapeField) Size() int {
+ var freqSize int
+ if n.frequencies != nil {
+ freqSize = n.frequencies.Size()
+ }
  return reflectStaticSizeGeoShapeField + size.SizeOfPtr +
  len(n.name) +
- len(n.arrayPositions)*size.SizeOfUint64
+ len(n.arrayPositions)*size.SizeOfUint64 +
+ len(n.encodedValue) +
+ len(n.value) +
+ freqSize
 }
 
 func (n *GeoShapeField) Name() string {

document/field_ip.go

@@ -44,10 +44,15 @@ type IPField struct {
 }
 
 func (b *IPField) Size() int {
+ var freqSize int
+ if b.frequencies != nil {
+ freqSize = b.frequencies.Size()
+ }
  return reflectStaticSizeIPField + size.SizeOfPtr +
  len(b.name) +
  len(b.arrayPositions)*size.SizeOfUint64 +
- len(b.value)
+ len(b.value) +
+ freqSize
 }
 
 func (b *IPField) Name() string {

document/field_numeric.go

@@ -46,9 +46,15 @@ type NumericField struct {
 }
 
 func (n *NumericField) Size() int {
+ var freqSize int
+ if n.frequencies != nil {
+ freqSize = n.frequencies.Size()
+ }
  return reflectStaticSizeNumericField + size.SizeOfPtr +
  len(n.name) +
- len(n.arrayPositions)*size.SizeOfPtr
+ len(n.arrayPositions)*size.SizeOfUint64 +
+ len(n.value) +
+ freqSize
 }
 
 func (n *NumericField) Name() string {

document/field_text.go

@@ -44,10 +44,15 @@ type TextField struct {
 }
 
 func (t *TextField) Size() int {
+ var freqSize int
+ if t.frequencies != nil {
+ freqSize = t.frequencies.Size()
+ }
  return reflectStaticSizeTextField + size.SizeOfPtr +
  len(t.name) +
  len(t.arrayPositions)*size.SizeOfUint64 +
- len(t.value)
+ len(t.value) +
+ freqSize
 }
 
 func (t *TextField) Name() string {

document/field_vector.go

@@ -47,6 +47,8 @@ type VectorField struct {
 func (n *VectorField) Size() int {
  return reflectStaticSizeVectorField + size.SizeOfPtr +
  len(n.name) +
+ len(n.similarity) +
+ len(n.vectorIndexOptimizedFor) +
  int(numBytesFloat32s(n.value))
 }
 

document/field_vector_base64.go

@@ -22,19 +22,33 @@ import (
  "encoding/binary"
  "fmt"
  "math"
+ "reflect"
 
  "github.com/blevesearch/bleve/v2/size"
  "github.com/blevesearch/bleve/v2/util"
  index "github.com/blevesearch/bleve_index_api"
 )
 
+var reflectStaticSizeVectorBase64Field int
+
+func init() {
+ var f VectorBase64Field
+ reflectStaticSizeVectorBase64Field = int(reflect.TypeOf(f).Size())
+}
+
 type VectorBase64Field struct {
  vectorField *VectorField
  base64Encoding string
 }
 
 func (n *VectorBase64Field) Size() int {
- return n.vectorField.Size()
+ var vecFieldSize int
+ if n.vectorField != nil {
+ vecFieldSize = n.vectorField.Size()
+ }
+ return reflectStaticSizeVectorBase64Field + size.SizeOfPtr +
+ len(n.base64Encoding) +
+ vecFieldSize
 }
 
 func (n *VectorBase64Field) Name() string {

index/scorch/snapshot_index_vr.go

@@ -52,7 +52,9 @@ type IndexSnapshotVectorReader struct {
 
 func (i *IndexSnapshotVectorReader) Size() int {
  sizeInBytes := reflectStaticSizeIndexSnapshotVectorReader + size.SizeOfPtr +
- len(i.vector) + len(i.field) + len(i.currID)
+ len(i.vector)*size.SizeOfFloat32 +
+ len(i.field) +
+ len(i.currID)
 
  for _, entry := range i.postings {
  sizeInBytes += entry.Size()

search/scorer/scorer_knn.go

@@ -47,7 +47,8 @@ type KNNQueryScorer struct {
 
 func (s *KNNQueryScorer) Size() int {
  sizeInBytes := reflectStaticSizeKNNQueryScorer + size.SizeOfPtr +
- (len(s.queryVector) * size.SizeOfFloat32) + len(s.queryField)
+ (len(s.queryVector) * size.SizeOfFloat32) + len(s.queryField) +
+ len(s.similarityMetric)
 
  if s.queryWeightExplanation != nil {
  sizeInBytes += s.queryWeightExplanation.Size()