diff --git a/cmd/es-rollover/app/flags.go b/cmd/es-rollover/app/flags.go index bf7bfcac1f0..58d0fd7c432 100644 --- a/cmd/es-rollover/app/flags.go +++ b/cmd/es-rollover/app/flags.go @@ -10,29 +10,31 @@ import ( ) const ( - indexPrefix = "index-prefix" - archive = "archive" - username = "es.username" - password = "es.password" - useILM = "es.use-ilm" - ilmPolicyName = "es.ilm-policy-name" - timeout = "timeout" - skipDependencies = "skip-dependencies" - adaptiveSampling = "adaptive-sampling" + indexPrefix = "index-prefix" + archive = "archive" + username = "es.username" + password = "es.password" + useILM = "es.use-ilm" + ilmPolicyName = "es.ilm-policy-name" + timeout = "timeout" + skipDependencies = "skip-dependencies" + adaptiveSampling = "adaptive-sampling" + disableLogsFieldSearch = "disable-logs-field-search" ) // Config holds the global configurations for the es rollover, common to all actions type Config struct { - IndexPrefix string - Archive bool - Username string - Password string - TLSEnabled bool - ILMPolicyName string - UseILM bool - Timeout int - SkipDependencies bool - AdaptiveSampling bool + IndexPrefix string + Archive bool + Username string + Password string + TLSEnabled bool + ILMPolicyName string + UseILM bool + Timeout int + SkipDependencies bool + AdaptiveSampling bool + DisableLogsFieldSearch bool } // AddFlags adds flags @@ -46,6 +48,7 @@ func AddFlags(flags *flag.FlagSet) { flags.Int(timeout, 120, "Number of seconds to wait for master node response") flags.Bool(skipDependencies, false, "Disable rollover for dependencies index") flags.Bool(adaptiveSampling, false, "Enable rollover for adaptive sampling index") + flags.Bool(disableLogsFieldSearch, false, "Set to `true` to set type logs.field mapping to object") } // InitFromViper initializes config from viper.Viper. @@ -62,4 +65,5 @@ func (c *Config) InitFromViper(v *viper.Viper) { c.Timeout = v.GetInt(timeout) c.SkipDependencies = v.GetBool(skipDependencies) c.AdaptiveSampling = v.GetBool(adaptiveSampling) + c.DisableLogsFieldSearch = v.GetBool(disableLogsFieldSearch) } diff --git a/cmd/es-rollover/app/flags_test.go b/cmd/es-rollover/app/flags_test.go index e2a2b444a6c..40e0ccab824 100644 --- a/cmd/es-rollover/app/flags_test.go +++ b/cmd/es-rollover/app/flags_test.go @@ -32,6 +32,7 @@ func TestBindFlags(t *testing.T) { "--es.ilm-policy-name=jaeger-ilm", "--skip-dependencies=true", "--adaptive-sampling=true", + "--disable-logs-field-search=true", }) require.NoError(t, err) @@ -44,4 +45,5 @@ func TestBindFlags(t *testing.T) { assert.Equal(t, "jaeger-ilm", c.ILMPolicyName) assert.True(t, c.SkipDependencies) assert.True(t, c.AdaptiveSampling) + assert.True(t, c.DisableLogsFieldSearch) } diff --git a/cmd/es-rollover/app/init/action.go b/cmd/es-rollover/app/init/action.go index dcb7ad4735c..4ebdaac034f 100644 --- a/cmd/es-rollover/app/init/action.go +++ b/cmd/es-rollover/app/init/action.go @@ -40,6 +40,7 @@ func (c Action) getMapping(version uint, templateName string) (string, error) { UseILM: c.Config.UseILM, ILMPolicyName: c.Config.ILMPolicyName, EsVersion: version, + DisableLogsFieldSearch: c.Config.DisableLogsFieldSearch, } return mappingBuilder.GetMapping(templateName) } diff --git a/cmd/esmapping-generator/app/flags.go b/cmd/esmapping-generator/app/flags.go index 4c684716044..6434edac5eb 100644 --- a/cmd/esmapping-generator/app/flags.go +++ b/cmd/esmapping-generator/app/flags.go @@ -9,23 +9,25 @@ import ( // Options represent configurable parameters for jaeger-esmapping-generator type Options struct { - Mapping string - EsVersion uint - Shards int64 - Replicas int64 - IndexPrefix string - UseILM string // using string as util is being used in python and using bool leads to type issues. - ILMPolicyName string + Mapping string + EsVersion uint + Shards int64 + Replicas int64 + IndexPrefix string + UseILM string // using string as util is being used in python and using bool leads to type issues. + ILMPolicyName string + DisableLogsFieldSearch string // using string as util is being used in python and using bool leads to type issues. } const ( - mappingFlag = "mapping" - esVersionFlag = "es-version" - shardsFlag = "shards" - replicasFlag = "replicas" - indexPrefixFlag = "index-prefix" - useILMFlag = "use-ilm" - ilmPolicyNameFlag = "ilm-policy-name" + mappingFlag = "mapping" + esVersionFlag = "es-version" + shardsFlag = "shards" + replicasFlag = "replicas" + indexPrefixFlag = "index-prefix" + useILMFlag = "use-ilm" + ilmPolicyNameFlag = "ilm-policy-name" + disableLogsFieldSearch = "disable-logs-field-search" ) // AddFlags adds flags for esmapping-generator main program @@ -65,6 +67,11 @@ func (o *Options) AddFlags(command *cobra.Command) { ilmPolicyNameFlag, "jaeger-ilm-policy", "The name of the ILM policy to use if ILM is active") + command.Flags().StringVar( + &o.DisableLogsFieldSearch, + disableLogsFieldSearch, + "false", + "Set to `true` to set type logs.field mapping to object") // mark mapping flag as mandatory command.MarkFlagRequired(mappingFlag) diff --git a/cmd/esmapping-generator/app/flags_test.go b/cmd/esmapping-generator/app/flags_test.go index 5189685b4d6..ff616d1c4a7 100644 --- a/cmd/esmapping-generator/app/flags_test.go +++ b/cmd/esmapping-generator/app/flags_test.go @@ -25,6 +25,7 @@ func TestOptionsWithDefaultFlags(t *testing.T) { assert.Equal(t, "", o.IndexPrefix) assert.Equal(t, "false", o.UseILM) assert.Equal(t, "jaeger-ilm-policy", o.ILMPolicyName) + assert.Equal(t, "false", o.DisableLogsFieldSearch) } func TestOptionsWithFlags(t *testing.T) { @@ -40,6 +41,7 @@ func TestOptionsWithFlags(t *testing.T) { "--index-prefix=test", "--use-ilm=true", "--ilm-policy-name=jaeger-test-policy", + "--disable-logs-field-search=true", }) require.NoError(t, err) assert.Equal(t, "jaeger-span", o.Mapping) @@ -49,6 +51,7 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "test", o.IndexPrefix) assert.Equal(t, "true", o.UseILM) assert.Equal(t, "jaeger-test-policy", o.ILMPolicyName) + assert.Equal(t, "true", o.DisableLogsFieldSearch) } func TestMain(m *testing.M) { diff --git a/cmd/esmapping-generator/app/renderer/render.go b/cmd/esmapping-generator/app/renderer/render.go index 7cb3c89f49a..531162c0bf9 100644 --- a/cmd/esmapping-generator/app/renderer/render.go +++ b/cmd/esmapping-generator/app/renderer/render.go @@ -24,15 +24,20 @@ func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, e if err != nil { return "", err } + disableLogsFieldSearch, err := strconv.ParseBool(opt.DisableLogsFieldSearch) + if err != nil { + return "", err + } mappingBuilder := mappings.MappingBuilder{ - TemplateBuilder: builder, - Shards: opt.Shards, - Replicas: opt.Replicas, - EsVersion: opt.EsVersion, - IndexPrefix: opt.IndexPrefix, - UseILM: enableILM, - ILMPolicyName: opt.ILMPolicyName, + TemplateBuilder: builder, + Shards: opt.Shards, + Replicas: opt.Replicas, + EsVersion: opt.EsVersion, + IndexPrefix: opt.IndexPrefix, + UseILM: enableILM, + ILMPolicyName: opt.ILMPolicyName, + DisableLogsFieldSearch: disableLogsFieldSearch, } return mappingBuilder.GetMapping(opt.Mapping) } diff --git a/cmd/esmapping-generator/app/renderer/render_test.go b/cmd/esmapping-generator/app/renderer/render_test.go index aadaf1c5125..a1e3024da9e 100644 --- a/cmd/esmapping-generator/app/renderer/render_test.go +++ b/cmd/esmapping-generator/app/renderer/render_test.go @@ -15,6 +15,7 @@ import ( "github.com/jaegertracing/jaeger/cmd/esmapping-generator/app" "github.com/jaegertracing/jaeger/pkg/es/mocks" "github.com/jaegertracing/jaeger/pkg/testutils" + "github.com/jaegertracing/jaeger/plugin/storage/es/mappings" ) func TestIsValidOption(t *testing.T) { @@ -36,22 +37,86 @@ func TestIsValidOption(t *testing.T) { func Test_getMappingAsString(t *testing.T) { tests := []struct { - name string - args app.Options - want string - wantErr error + name string + args app.Options + wantedMapping *mappings.MappingBuilder + want string + wantErr error }{ { - name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"}, + name: "ES version 7", + args: app.Options{ + Mapping: "jaeger-span", + EsVersion: 7, + Shards: 5, + Replicas: 1, + IndexPrefix: "test-", + UseILM: "true", + ILMPolicyName: "jaeger-test-policy", + DisableLogsFieldSearch: "true", + }, + wantedMapping: &mappings.MappingBuilder{ + EsVersion: 7, + Shards: 5, + Replicas: 1, + IndexPrefix: "test-", + UseILM: true, + ILMPolicyName: "jaeger-test-policy", + DisableLogsFieldSearch: true, + }, want: "ES version 7", }, { - name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true", ILMPolicyName: "jaeger-test-policy"}, + name: "Parse Error version 7", + args: app.Options{ + Mapping: "jaeger-span", + EsVersion: 7, + Shards: 5, + Replicas: 1, + IndexPrefix: "test-", + UseILM: "true", + ILMPolicyName: "jaeger-test-policy", + DisableLogsFieldSearch: "false", + }, + wantedMapping: &mappings.MappingBuilder{ + EsVersion: 7, + Shards: 5, + Replicas: 1, + IndexPrefix: "test-", + UseILM: false, + ILMPolicyName: "jaeger-test-policy", + DisableLogsFieldSearch: false, + }, wantErr: errors.New("parse error"), }, { - name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo", ILMPolicyName: "jaeger-test-policy"}, - wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"), + name: "Parse bool error for UseILM", + args: app.Options{ + Mapping: "jaeger-span", + EsVersion: 7, + Shards: 5, + Replicas: 1, + IndexPrefix: "test-", + UseILM: "foo", + ILMPolicyName: "jaeger-test-policy", + }, + wantedMapping: &mappings.MappingBuilder{}, + wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"), + }, + { + name: "Parse bool error for DisableLogsFieldSearch", + args: app.Options{ + Mapping: "jaeger-span", + EsVersion: 7, + Shards: 5, + Replicas: 1, + IndexPrefix: "test-", + UseILM: "false", + ILMPolicyName: "jaeger-test-policy", + DisableLogsFieldSearch: "foo", + }, + wantedMapping: &mappings.MappingBuilder{}, + wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"), }, } for _, tt := range tests { @@ -60,11 +125,17 @@ func Test_getMappingAsString(t *testing.T) { mockTemplateApplier := &mocks.TemplateApplier{} mockTemplateApplier.On("Execute", mock.Anything, mock.Anything).Return( func(wr io.Writer, data any) error { + if actualMapping, ok := data.(*mappings.MappingBuilder); ok { + require.Equal(t, tt.wantedMapping, actualMapping) + } else { + require.Fail(t, "unexpected mapping type") + } wr.Write([]byte(tt.want)) return nil }, ) mockTemplateBuilder := &mocks.TemplateBuilder{} + tt.wantedMapping.TemplateBuilder = mockTemplateBuilder mockTemplateBuilder.On("Parse", mock.Anything).Return(mockTemplateApplier, tt.wantErr) // Test diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index 129e10c4dc9..44c9ba3c4e1 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -75,6 +75,7 @@ type Configuration struct { Version uint `mapstructure:"version"` LogLevel string `mapstructure:"log_level"` SendGetBodyAs string `mapstructure:"send_get_body_as"` + DisableLogsFieldSearch bool `mapstructure:"disable_logs_field_search"` } // TagsAsFields holds configuration for tag schema. diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go index 1a9874dc0c6..bf8e204085b 100644 --- a/plugin/storage/es/factory.go +++ b/plugin/storage/es/factory.go @@ -236,6 +236,7 @@ func createSpanReader( Logger: logger, MetricsFactory: mFactory, Tracer: tp.Tracer("esSpanStore.SpanReader"), + DisableLogsFieldSearch: cfg.DisableLogsFieldSearch, }), nil } @@ -322,6 +323,7 @@ func mappingBuilderFromConfig(cfg *config.Configuration) mappings.MappingBuilder PrioritySpanTemplate: cfg.PrioritySpanTemplate, PriorityServiceTemplate: cfg.PriorityServiceTemplate, PriorityDependenciesTemplate: cfg.PriorityDependenciesTemplate, + DisableLogsFieldSearch: cfg.DisableLogsFieldSearch, } } diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-6.json b/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-6.json new file mode 100644 index 00000000000..f47b81ad2a0 --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-6.json @@ -0,0 +1,165 @@ +{ + "template": "*jaeger-span-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true, + "index.mapper.dynamic":false + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":false + }, + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ] + }, + "span":{ + "properties":{ + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "parentSpanID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + }, + "startTime":{ + "type":"long" + }, + "startTimeMillis":{ + "type":"date", + "format":"epoch_millis" + }, + "duration":{ + "type":"long" + }, + "flags":{ + "type":"integer" + }, + "logs":{ + "type":"object", + "dynamic":false, + "properties":{ + "timestamp":{ + "type":"long" + }, + "fields":{ + "type":"object", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "process":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "references":{ + "type":"nested", + "dynamic":false, + "properties":{ + "refType":{ + "type":"keyword", + "ignore_above":256 + }, + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + } + } +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-7.json b/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-7.json new file mode 100644 index 00000000000..6559ab266ee --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-7.json @@ -0,0 +1,164 @@ +{ + "index_patterns": "*test-jaeger-span-*", + "aliases": { + "test-jaeger-span-read": {} + }, + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + ,"lifecycle": { + "name": "jaeger-test-policy", + "rollover_alias": "test-jaeger-span-write" + } + }, + "mappings":{ + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ], + "properties":{ + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "parentSpanID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + }, + "startTime":{ + "type":"long" + }, + "startTimeMillis":{ + "type":"date", + "format":"epoch_millis" + }, + "duration":{ + "type":"long" + }, + "flags":{ + "type":"integer" + }, + "logs":{ + "type":"object", + "dynamic":false, + "properties":{ + "timestamp":{ + "type":"long" + }, + "fields":{ + "type":"object", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "process":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "references":{ + "type":"nested", + "dynamic":false, + "properties":{ + "refType":{ + "type":"keyword", + "ignore_above":256 + }, + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + } +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-8.json b/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-8.json new file mode 100644 index 00000000000..a69b7b60a7a --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-span-with-logs-fieldtype-object-8.json @@ -0,0 +1,167 @@ +{ + "priority": 500, + "index_patterns": "test-jaeger-span-*", + "template": { + "aliases": { + "test-jaeger-span-read": {} + }, + "settings": { + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit": 50, + "index.requests.cache.enable": true, + "lifecycle": { + "name": "jaeger-test-policy", + "rollover_alias": "test-jaeger-span-write" + } + }, + "mappings": { + "dynamic_templates": [ + { + "span_tags_map": { + "mapping": { + "type": "keyword", + "ignore_above": 256 + }, + "path_match": "tag.*" + } + }, + { + "process_tags_map": { + "mapping": { + "type": "keyword", + "ignore_above": 256 + }, + "path_match": "process.tag.*" + } + } + ], + "properties": { + "traceID": { + "type": "keyword", + "ignore_above": 256 + }, + "parentSpanID": { + "type": "keyword", + "ignore_above": 256 + }, + "spanID": { + "type": "keyword", + "ignore_above": 256 + }, + "operationName": { + "type": "keyword", + "ignore_above": 256 + }, + "startTime": { + "type": "long" + }, + "startTimeMillis": { + "type": "date", + "format": "epoch_millis" + }, + "duration": { + "type": "long" + }, + "flags": { + "type": "integer" + }, + "logs": { + "type": "object", + "dynamic": false, + "properties": { + "timestamp": { + "type": "long" + }, + "fields": { + "type": "object", + "dynamic": false, + "properties": { + "key": { + "type": "keyword", + "ignore_above": 256 + }, + "value": { + "type": "keyword", + "ignore_above": 256 + }, + "tagType": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "process": { + "properties": { + "serviceName": { + "type": "keyword", + "ignore_above": 256 + }, + "tag": { + "type": "object" + }, + "tags": { + "type": "nested", + "dynamic": false, + "properties": { + "key": { + "type": "keyword", + "ignore_above": 256 + }, + "value": { + "type": "keyword", + "ignore_above": 256 + }, + "tagType": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + }, + "references": { + "type": "nested", + "dynamic": false, + "properties": { + "refType": { + "type": "keyword", + "ignore_above": 256 + }, + "traceID": { + "type": "keyword", + "ignore_above": 256 + }, + "spanID": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tag": { + "type": "object" + }, + "tags": { + "type": "nested", + "dynamic": false, + "properties": { + "key": { + "type": "keyword", + "ignore_above": 256 + }, + "value": { + "type": "keyword", + "ignore_above": 256 + }, + "tagType": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } +} diff --git a/plugin/storage/es/mappings/jaeger-span-6.json b/plugin/storage/es/mappings/jaeger-span-6.json index 0df36baae8e..ec3b0d2a320 100644 --- a/plugin/storage/es/mappings/jaeger-span-6.json +++ b/plugin/storage/es/mappings/jaeger-span-6.json @@ -65,14 +65,22 @@ "type":"integer" }, "logs":{ + {{- if .DisableLogsFieldSearch }} + "type":"object", + {{- else }} "type":"nested", + {{- end }} "dynamic":false, "properties":{ "timestamp":{ "type":"long" }, "fields":{ + {{- if .DisableLogsFieldSearch }} + "type":"object", + {{- else }} "type":"nested", + {{- end }} "dynamic":false, "properties":{ "key":{ diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json index 3e8f7c39358..c322317638f 100644 --- a/plugin/storage/es/mappings/jaeger-span-7.json +++ b/plugin/storage/es/mappings/jaeger-span-7.json @@ -69,14 +69,22 @@ "type":"integer" }, "logs":{ + {{- if .DisableLogsFieldSearch }} + "type":"object", + {{- else }} "type":"nested", + {{- end }} "dynamic":false, "properties":{ "timestamp":{ "type":"long" }, "fields":{ + {{- if .DisableLogsFieldSearch }} + "type":"object", + {{- else }} "type":"nested", + {{- end }} "dynamic":false, "properties":{ "key":{ diff --git a/plugin/storage/es/mappings/jaeger-span-8.json b/plugin/storage/es/mappings/jaeger-span-8.json index d91fadba619..5ae58122b17 100644 --- a/plugin/storage/es/mappings/jaeger-span-8.json +++ b/plugin/storage/es/mappings/jaeger-span-8.json @@ -72,14 +72,22 @@ "type": "integer" }, "logs": { + {{- if .DisableLogsFieldSearch }} + "type": "object", + {{- else }} "type": "nested", + {{- end }} "dynamic": false, "properties": { "timestamp": { "type": "long" }, "fields": { + {{- if .DisableLogsFieldSearch }} + "type": "object", + {{- else }} "type": "nested", + {{- end }} "dynamic": false, "properties": { "key": { diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go index 1962f85100d..65d010a29d5 100644 --- a/plugin/storage/es/mappings/mapping.go +++ b/plugin/storage/es/mappings/mapping.go @@ -29,6 +29,7 @@ type MappingBuilder struct { IndexPrefix string UseILM bool ILMPolicyName string + DisableLogsFieldSearch bool } // GetMapping returns the rendered mapping based on elasticsearch version diff --git a/plugin/storage/es/mappings/mapping_test.go b/plugin/storage/es/mappings/mapping_test.go index 194a15b2c24..fed42819331 100644 --- a/plugin/storage/es/mappings/mapping_test.go +++ b/plugin/storage/es/mappings/mapping_test.go @@ -6,7 +6,6 @@ package mappings import ( "embed" "errors" - "fmt" "io" "os" "testing" @@ -31,21 +30,26 @@ func TestMappingBuilderGetMapping(t *testing.T) { jaegerDependencies = "jaeger-dependencies" ) tests := []struct { - mapping string - esVersion uint + mapping string + esVersion uint + fixtureName string + disableLogsFieldSearch bool }{ - {mapping: jaegerSpan, esVersion: 8}, - {mapping: jaegerSpan, esVersion: 7}, - {mapping: jaegerSpan, esVersion: 6}, - {mapping: jaegerService, esVersion: 8}, - {mapping: jaegerService, esVersion: 7}, - {mapping: jaegerService, esVersion: 6}, - {mapping: jaegerDependencies, esVersion: 8}, - {mapping: jaegerDependencies, esVersion: 7}, - {mapping: jaegerDependencies, esVersion: 6}, + {mapping: jaegerSpan, esVersion: 8, fixtureName: "jaeger-span-8"}, + {mapping: jaegerSpan, esVersion: 7, fixtureName: "jaeger-span-7"}, + {mapping: jaegerSpan, esVersion: 6, fixtureName: "jaeger-span-6"}, + {mapping: jaegerSpan, esVersion: 8, fixtureName: "jaeger-span-with-logs-fieldtype-object-8", disableLogsFieldSearch: true}, + {mapping: jaegerSpan, esVersion: 7, fixtureName: "jaeger-span-with-logs-fieldtype-object-7", disableLogsFieldSearch: true}, + {mapping: jaegerSpan, esVersion: 6, fixtureName: "jaeger-span-with-logs-fieldtype-object-6", disableLogsFieldSearch: true}, + {mapping: jaegerService, esVersion: 8, fixtureName: "jaeger-service-8"}, + {mapping: jaegerService, esVersion: 7, fixtureName: "jaeger-service-7"}, + {mapping: jaegerService, esVersion: 6, fixtureName: "jaeger-service-6"}, + {mapping: jaegerDependencies, esVersion: 8, fixtureName: "jaeger-dependencies-8"}, + {mapping: jaegerDependencies, esVersion: 7, fixtureName: "jaeger-dependencies-7"}, + {mapping: jaegerDependencies, esVersion: 6, fixtureName: "jaeger-dependencies-6"}, } for _, tt := range tests { - t.Run(tt.mapping, func(t *testing.T) { + t.Run(tt.fixtureName, func(t *testing.T) { mb := &MappingBuilder{ TemplateBuilder: es.TextTemplateBuilder{}, Shards: 3, @@ -57,12 +61,12 @@ func TestMappingBuilderGetMapping(t *testing.T) { IndexPrefix: "test-", UseILM: true, ILMPolicyName: "jaeger-test-policy", + DisableLogsFieldSearch: tt.disableLogsFieldSearch, } got, err := mb.GetMapping(tt.mapping) require.NoError(t, err) var wantbytes []byte - fileSuffix := fmt.Sprintf("-%d", tt.esVersion) - wantbytes, err = FIXTURES.ReadFile("fixtures/" + tt.mapping + fileSuffix + ".json") + wantbytes, err = FIXTURES.ReadFile("fixtures/" + tt.fixtureName + ".json") require.NoError(t, err) want := string(wantbytes) assert.Equal(t, want, got) @@ -108,7 +112,8 @@ func TestMappingBuilderFixMapping(t *testing.T) { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(nil) - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) return &tb }, err: "", @@ -119,7 +124,8 @@ func TestMappingBuilderFixMapping(t *testing.T) { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template exec error")) - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) return &tb }, err: "template exec error", @@ -185,7 +191,8 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(nil) - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) return &tb }, err: "", @@ -205,7 +212,8 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) return &tb }, err: "template load error", @@ -225,7 +233,8 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(nil) - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) return &tb }, err: "", @@ -245,7 +254,8 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) return &tb }, err: "template load error", @@ -264,7 +274,8 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) return &tb }, err: "template load error", @@ -283,7 +294,8 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) return &tb }, err: "template load error", @@ -314,7 +326,8 @@ func TestMappingBuilderGetDependenciesMappings(t *testing.T) { tb := mocks.TemplateBuilder{} ta := mocks.TemplateApplier{} ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) - tb.On("Parse", mock.Anything).Return(&ta, nil) + tb.On("ParseFieldType", mock.Anything).Return(&ta, nil) + tb.On("Parse", mock.Anything).Times(2).Return(&ta, nil) mappingBuilder := MappingBuilder{ TemplateBuilder: &tb, diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 23ae110803e..5024352f0aa 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -57,6 +57,7 @@ const ( suffixMaxDocCount = ".max-doc-count" suffixLogLevel = ".log-level" suffixSendGetBodyAs = ".send-get-body-as" + suffixDisableLogsFieldSearch = ".disable-logs-field-search" // default number of documents to return from a query (elasticsearch allowed limit) // see search.max_buckets and index.max_result_window defaultMaxDocCount = 10_000 @@ -277,6 +278,13 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { nsConfig.namespace+suffixAdaptiveSamplingLookback, nsConfig.AdaptiveSamplingLookback, "How far back to look for the latest adaptive sampling probabilities") + flagSet.Bool( + nsConfig.namespace+suffixDisableLogsFieldSearch, + nsConfig.DisableLogsFieldSearch, + "(experimental) When true, turns off the ability to search within log fields, and uses "+ + "more efficient 'object' mapping for logs[].fields[], instead of 'nested' mapping.", + ) + if nsConfig.namespace == archiveNamespace { flagSet.Bool( nsConfig.namespace+suffixEnabled, @@ -356,6 +364,8 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { // Dependencies calculation should be daily, and this index size is very small cfg.IndexDateLayoutDependencies = initDateLayout(defaultIndexRolloverFrequency, separator) + cfg.DisableLogsFieldSearch = v.GetBool(cfg.namespace + suffixDisableLogsFieldSearch) + var err error cfg.TLS, err = cfg.getTLSFlagsConfig().InitFromViper(v) if err != nil { diff --git a/plugin/storage/es/options_test.go b/plugin/storage/es/options_test.go index 8e08c74a6f1..71b7b179f69 100644 --- a/plugin/storage/es/options_test.go +++ b/plugin/storage/es/options_test.go @@ -67,6 +67,7 @@ func TestOptionsWithFlags(t *testing.T) { "--es.tags-as-fields.dot-replacement=!", "--es.use-ilm=true", "--es.send-get-body-as=POST", + "--es.disable-logs-field-search=true", }) require.NoError(t, err) opts.InitFromViper(v) @@ -105,6 +106,7 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, "2006.01.02.15", aux.IndexDateLayoutSpans) assert.True(t, primary.UseILM) assert.Equal(t, "POST", aux.SendGetBodyAs) + assert.True(t, primary.DisableLogsFieldSearch) } func TestEmptyRemoteReadClusters(t *testing.T) { diff --git a/plugin/storage/es/spanstore/fixtures/query_04.json b/plugin/storage/es/spanstore/fixtures/query_04.json new file mode 100644 index 00000000000..4477ab8eb5c --- /dev/null +++ b/plugin/storage/es/spanstore/fixtures/query_04.json @@ -0,0 +1,78 @@ +{ + "bool":{ + "should":[ + { + "bool":{ + "must":{ + "regexp":{ + "tag.bat@foo":{ + "value":"spook" + } + } + } + } + }, + { + "bool":{ + "must":{ + "regexp":{ + "process.tag.bat@foo":{ + "value":"spook" + } + } + } + } + }, + { + "nested":{ + "path":"tags", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "tags.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "tags.value":{ + "value":"spook" + } + } + } + ] + } + } + } + }, + { + "nested":{ + "path":"process.tags", + "query":{ + "bool":{ + "must":[ + { + "match":{ + "process.tags.key":{ + "query":"bat.foo" + } + } + }, + { + "regexp":{ + "process.tags.value":{ + "value":"spook" + } + } + } + ] + } + } + } + } + ] + } +} diff --git a/plugin/storage/es/spanstore/reader.go b/plugin/storage/es/spanstore/reader.go index 2d856c19614..8cd67867760 100644 --- a/plugin/storage/es/spanstore/reader.go +++ b/plugin/storage/es/spanstore/reader.go @@ -76,7 +76,8 @@ var ( objectTagFieldList = []string{objectTagsField, objectProcessTagsField} - nestedTagFieldList = []string{nestedTagsField, nestedProcessTagsField, nestedLogFieldsField} + nestedTagFieldList = []string{nestedTagsField, nestedProcessTagsField, nestedLogFieldsField} + nestedTagFieldListWithoutLogsFields = []string{nestedTagsField, nestedProcessTagsField} ) // SpanReader can query for and load traces from ElasticSearch @@ -99,6 +100,7 @@ type SpanReader struct { useReadWriteAliases bool logger *zap.Logger tracer trace.Tracer + DisableLogsFieldSearch bool } // SpanReaderParams holds constructor params for NewSpanReader @@ -115,6 +117,7 @@ type SpanReaderParams struct { Archive bool UseReadWriteAliases bool RemoteReadClusters []string + DisableLogsFieldSearch bool MetricsFactory metrics.Factory Logger *zap.Logger Tracer trace.Tracer @@ -145,6 +148,7 @@ func NewSpanReader(p SpanReaderParams) *SpanReader { useReadWriteAliases: p.UseReadWriteAliases, logger: p.Logger, tracer: p.Tracer, + DisableLogsFieldSearch: p.DisableLogsFieldSearch, } } @@ -660,13 +664,17 @@ func (*SpanReader) buildOperationNameQuery(operationName string) elastic.Query { func (s *SpanReader) buildTagQuery(k string, v string) elastic.Query { objectTagListLen := len(objectTagFieldList) - queries := make([]elastic.Query, len(nestedTagFieldList)+objectTagListLen) + nestedFields := nestedTagFieldList + if s.DisableLogsFieldSearch { + nestedFields = nestedTagFieldListWithoutLogsFields + } + queries := make([]elastic.Query, len(nestedFields)+objectTagListLen) kd := s.spanConverter.ReplaceDot(k) for i := range objectTagFieldList { queries[i] = s.buildObjectQuery(objectTagFieldList[i], kd, v) } - for i := range nestedTagFieldList { - queries[i+objectTagListLen] = s.buildNestedQuery(nestedTagFieldList[i], k, v) + for i := range nestedFields { + queries[i+objectTagListLen] = s.buildNestedQuery(nestedFields[i], k, v) } // but configuration can change over time diff --git a/plugin/storage/es/spanstore/reader_test.go b/plugin/storage/es/spanstore/reader_test.go index c96c7858d8b..c834e769c01 100644 --- a/plugin/storage/es/spanstore/reader_test.go +++ b/plugin/storage/es/spanstore/reader_test.go @@ -118,6 +118,30 @@ func withSpanReader(t *testing.T, fn func(r *spanReaderTest)) { fn(r) } +func withDisableLogsFieldSearchSpanReader(t *testing.T, fn func(r *spanReaderTest)) { + client := &mocks.Client{} + tracer, exp, closer := tracerProvider(t) + defer closer() + logger, logBuffer := testutils.NewLogger() + r := &spanReaderTest{ + client: client, + logger: logger, + logBuffer: logBuffer, + traceBuffer: exp, + reader: NewSpanReader(SpanReaderParams{ + Client: func() es.Client { return client }, + Logger: zap.NewNop(), + Tracer: tracer.Tracer("test"), + MaxSpanAge: 0, + IndexPrefix: "", + TagDotReplacement: "@", + MaxDocCount: defaultMaxDocCount, + DisableLogsFieldSearch: true, + }), + } + fn(r) +} + func withArchiveSpanReader(t *testing.T, readAlias bool, fn func(r *spanReaderTest)) { client := &mocks.Client{} tracer, exp, closer := tracerProvider(t) @@ -764,6 +788,34 @@ func TestSpanReader_FindTraces(t *testing.T) { }) } +func TestSpanReader_FindTraces_WithDisableLogsFieldSearch(t *testing.T) { + goodAggregations := make(map[string]*json.RawMessage) + rawMessage := []byte(`{"buckets": []}`) + goodAggregations[traceIDAggregation] = (*json.RawMessage)(&rawMessage) + + withDisableLogsFieldSearchSpanReader(t, func(r *spanReaderTest) { + mockSearchService(r). + Return(&elastic.SearchResult{Aggregations: goodAggregations, Hits: nil}, nil) + mockMultiSearchService(r). + Return(&elastic.MultiSearchResult{ + Responses: []*elastic.SearchResult{}, + }, nil) + + traceQuery := &spanstore.TraceQueryParameters{ + ServiceName: serviceName, + Tags: map[string]string{ + "hello": "world", + }, + StartTimeMin: time.Now().Add(-1 * time.Hour), + StartTimeMax: time.Now(), + } + + traces, err := r.reader.FindTraces(context.Background(), traceQuery) + require.NoError(t, err) + assert.Empty(t, traces) + }) +} + func TestSpanReader_FindTracesInvalidQuery(t *testing.T) { goodAggregations := make(map[string]*json.RawMessage) rawMessage := []byte(`{"buckets": [{"key": "1","doc_count": 16},{"key": "2","doc_count": 16},{"key": "3","doc_count": 16}]}`) @@ -1193,6 +1245,21 @@ func TestSpanReader_buildTagQuery(t *testing.T) { }) } +func TestSpanReader_buildTagQuery_WithDisableLogsFieldSearch(t *testing.T) { + inStr, err := os.ReadFile("fixtures/query_04.json") + require.NoError(t, err) + withDisableLogsFieldSearchSpanReader(t, func(r *spanReaderTest) { + tagQuery := r.reader.buildTagQuery("bat.foo", "spook") + actual, err := tagQuery.Source() + require.NoError(t, err) + + expected := make(map[string]any) + json.Unmarshal(inStr, &expected) + + assert.EqualValues(t, expected, actual) + }) +} + func TestSpanReader_buildTagRegexQuery(t *testing.T) { inStr, err := os.ReadFile("fixtures/query_02.json") require.NoError(t, err)